2024-11-16 16:14:06,467 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 16:14:06,478 main DEBUG Took 0.009015 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 16:14:06,478 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 16:14:06,479 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 16:14:06,479 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 16:14:06,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,487 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 16:14:06,501 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,502 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,503 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,503 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,504 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,506 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,507 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,508 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,509 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,509 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,510 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,511 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,512 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,512 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,513 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,513 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,514 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 16:14:06,515 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,516 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 16:14:06,518 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 16:14:06,519 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 16:14:06,522 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 16:14:06,523 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 16:14:06,524 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 16:14:06,525 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 16:14:06,532 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 16:14:06,534 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 16:14:06,536 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 16:14:06,536 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 16:14:06,537 main DEBUG createAppenders(={Console}) 2024-11-16 16:14:06,538 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-16 16:14:06,538 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-16 16:14:06,538 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-16 16:14:06,539 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 16:14:06,539 main DEBUG OutputStream closed 2024-11-16 16:14:06,540 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 16:14:06,540 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 16:14:06,540 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-16 16:14:06,619 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 16:14:06,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 16:14:06,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 16:14:06,624 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 16:14:06,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 16:14:06,625 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 16:14:06,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 16:14:06,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 16:14:06,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 16:14:06,627 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 16:14:06,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 16:14:06,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 16:14:06,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 16:14:06,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 16:14:06,629 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 16:14:06,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 16:14:06,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 16:14:06,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 16:14:06,632 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 16:14:06,633 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-16 16:14:06,633 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 16:14:06,634 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-16T16:14:06,859 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883 2024-11-16 16:14:06,862 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 16:14:06,862 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T16:14:06,870 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-16T16:14:06,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-16T16:14:06,903 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7, deleteOnExit=true 2024-11-16T16:14:06,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-16T16:14:06,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/test.cache.data in system properties and HBase conf 2024-11-16T16:14:06,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.tmp.dir in system properties and HBase conf 2024-11-16T16:14:06,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.log.dir in system properties and HBase conf 2024-11-16T16:14:06,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-16T16:14:06,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-16T16:14:06,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-16T16:14:06,989 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T16:14:07,088 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T16:14:07,092 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-16T16:14:07,093 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-16T16:14:07,094 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-16T16:14:07,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T16:14:07,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-16T16:14:07,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-16T16:14:07,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-16T16:14:07,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T16:14:07,097 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-16T16:14:07,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/nfs.dump.dir in system properties and HBase conf 2024-11-16T16:14:07,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/java.io.tmpdir in system properties and HBase conf 2024-11-16T16:14:07,098 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-16T16:14:07,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-16T16:14:07,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-16T16:14:08,340 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-16T16:14:08,408 INFO [Time-limited test {}] log.Log(170): Logging initialized @2584ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-16T16:14:08,473 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T16:14:08,529 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T16:14:08,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T16:14:08,552 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T16:14:08,554 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T16:14:08,569 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T16:14:08,572 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.log.dir/,AVAILABLE} 2024-11-16T16:14:08,573 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T16:14:08,757 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/java.io.tmpdir/jetty-localhost-42417-hadoop-hdfs-3_4_1-tests_jar-_-any-15878718547835712654/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T16:14:08,764 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:42417} 2024-11-16T16:14:08,764 INFO [Time-limited test {}] server.Server(415): Started @2941ms 2024-11-16T16:14:09,374 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-16T16:14:09,384 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-16T16:14:09,385 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-16T16:14:09,385 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-16T16:14:09,385 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-16T16:14:09,386 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.log.dir/,AVAILABLE} 2024-11-16T16:14:09,387 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-16T16:14:09,484 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/java.io.tmpdir/jetty-localhost-34843-hadoop-hdfs-3_4_1-tests_jar-_-any-234312141113509724/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T16:14:09,485 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:34843} 2024-11-16T16:14:09,485 INFO [Time-limited test {}] server.Server(415): Started @3662ms 2024-11-16T16:14:09,532 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-16T16:14:10,727 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data1/current/BP-1064039519-172.17.0.2-1731773647624/current, will proceed with Du for space computation calculation, 2024-11-16T16:14:10,727 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data2/current/BP-1064039519-172.17.0.2-1731773647624/current, will proceed with Du for space computation calculation, 2024-11-16T16:14:10,754 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-16T16:14:10,796 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9947ee361db2236 with lease ID 0x86ad2d3fbda488e9: Processing first storage report for DS-8306a2d5-0da1-483b-a228-5fc711830e9a from datanode DatanodeRegistration(127.0.0.1:41113, datanodeUuid=b63cc791-161c-4efb-b0a1-2b19fdb310d8, infoPort=45187, infoSecurePort=0, ipcPort=33599, storageInfo=lv=-57;cid=testClusterID;nsid=1089588298;c=1731773647624) 2024-11-16T16:14:10,797 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9947ee361db2236 with lease ID 0x86ad2d3fbda488e9: from storage DS-8306a2d5-0da1-483b-a228-5fc711830e9a node DatanodeRegistration(127.0.0.1:41113, datanodeUuid=b63cc791-161c-4efb-b0a1-2b19fdb310d8, infoPort=45187, infoSecurePort=0, ipcPort=33599, storageInfo=lv=-57;cid=testClusterID;nsid=1089588298;c=1731773647624), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-16T16:14:10,797 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf9947ee361db2236 with lease ID 0x86ad2d3fbda488e9: Processing first storage report for DS-a314e8e4-8f66-4415-87f7-57624b092f15 from datanode DatanodeRegistration(127.0.0.1:41113, datanodeUuid=b63cc791-161c-4efb-b0a1-2b19fdb310d8, infoPort=45187, infoSecurePort=0, ipcPort=33599, storageInfo=lv=-57;cid=testClusterID;nsid=1089588298;c=1731773647624) 2024-11-16T16:14:10,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf9947ee361db2236 with lease ID 0x86ad2d3fbda488e9: from storage DS-a314e8e4-8f66-4415-87f7-57624b092f15 node DatanodeRegistration(127.0.0.1:41113, datanodeUuid=b63cc791-161c-4efb-b0a1-2b19fdb310d8, infoPort=45187, infoSecurePort=0, ipcPort=33599, storageInfo=lv=-57;cid=testClusterID;nsid=1089588298;c=1731773647624), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-16T16:14:10,855 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883 2024-11-16T16:14:10,923 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/zookeeper_0, clientPort=64212, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T16:14:10,931 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64212 2024-11-16T16:14:10,941 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:10,944 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:11,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741825_1001 (size=7) 2024-11-16T16:14:11,567 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 with version=8 2024-11-16T16:14:11,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/hbase-staging 2024-11-16T16:14:11,659 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T16:14:11,935 INFO [Time-limited test {}] client.ConnectionUtils(128): master/303fe01965e8:0 server-side Connection retries=45 2024-11-16T16:14:11,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:11,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:11,952 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T16:14:11,952 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:11,953 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T16:14:12,098 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T16:14:12,158 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T16:14:12,169 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T16:14:12,173 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T16:14:12,201 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 4815 (auto-detected) 2024-11-16T16:14:12,203 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T16:14:12,232 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45907 2024-11-16T16:14:12,258 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45907 connecting to ZooKeeper ensemble=127.0.0.1:64212 2024-11-16T16:14:12,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459070x0, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T16:14:12,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45907-0x101446cd3870000 connected 2024-11-16T16:14:12,444 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:12,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:12,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T16:14:12,463 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050, hbase.cluster.distributed=false 2024-11-16T16:14:12,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T16:14:12,494 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45907 2024-11-16T16:14:12,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45907 2024-11-16T16:14:12,495 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45907 2024-11-16T16:14:12,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45907 2024-11-16T16:14:12,497 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45907 2024-11-16T16:14:12,603 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/303fe01965e8:0 server-side Connection retries=45 2024-11-16T16:14:12,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:12,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:12,605 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T16:14:12,605 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T16:14:12,606 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T16:14:12,608 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T16:14:12,610 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T16:14:12,611 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37355 2024-11-16T16:14:12,613 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37355 connecting to ZooKeeper ensemble=127.0.0.1:64212 2024-11-16T16:14:12,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:12,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:12,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373550x0, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T16:14:12,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37355-0x101446cd3870001 connected 2024-11-16T16:14:12,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T16:14:12,701 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T16:14:12,709 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T16:14:12,712 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T16:14:12,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T16:14:12,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-16T16:14:12,718 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37355 2024-11-16T16:14:12,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37355 2024-11-16T16:14:12,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-16T16:14:12,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37355 2024-11-16T16:14:12,739 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;303fe01965e8:45907 2024-11-16T16:14:12,740 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/303fe01965e8,45907,1731773651719 2024-11-16T16:14:12,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T16:14:12,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T16:14:12,816 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/303fe01965e8,45907,1731773651719 2024-11-16T16:14:12,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:12,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T16:14:12,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:12,879 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T16:14:12,881 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/303fe01965e8,45907,1731773651719 from backup master directory 2024-11-16T16:14:12,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T16:14:12,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/303fe01965e8,45907,1731773651719 2024-11-16T16:14:12,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T16:14:12,955 WARN [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T16:14:12,956 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=303fe01965e8,45907,1731773651719 2024-11-16T16:14:12,959 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T16:14:12,961 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T16:14:13,013 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/hbase.id] with ID: d498f81b-52a7-4718-ba95-d95378acab0e 2024-11-16T16:14:13,013 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/.tmp/hbase.id 2024-11-16T16:14:13,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741826_1002 (size=42) 2024-11-16T16:14:13,429 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/.tmp/hbase.id]:[hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/hbase.id] 2024-11-16T16:14:13,479 INFO [master/303fe01965e8:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-16T16:14:13,483 INFO [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T16:14:13,502 INFO [master/303fe01965e8:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-16T16:14:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:13,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:13,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741827_1003 (size=196) 2024-11-16T16:14:13,970 INFO [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:14:13,973 INFO [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T16:14:13,993 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:14,000 INFO [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T16:14:14,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741828_1004 (size=1189) 2024-11-16T16:14:14,454 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store 2024-11-16T16:14:14,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741829_1005 (size=34) 2024-11-16T16:14:14,876 INFO [master/303fe01965e8:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T16:14:14,879 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:14,881 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T16:14:14,881 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:14:14,881 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:14:14,883 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T16:14:14,883 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:14:14,883 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:14:14,885 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731773654881Disabling compacts and flushes for region at 1731773654881Disabling writes for close at 1731773654883 (+2 ms)Writing region close event to WAL at 1731773654883Closed at 1731773654883 2024-11-16T16:14:14,887 WARN [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/.initializing 2024-11-16T16:14:14,887 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/WALs/303fe01965e8,45907,1731773651719 2024-11-16T16:14:14,896 INFO [master/303fe01965e8:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T16:14:14,910 INFO [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C45907%2C1731773651719, suffix=, logDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/WALs/303fe01965e8,45907,1731773651719, archiveDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/oldWALs, maxLogs=10 2024-11-16T16:14:14,932 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/WALs/303fe01965e8,45907,1731773651719/303fe01965e8%2C45907%2C1731773651719.1731773654915, exclude list is [], retry=0 2024-11-16T16:14:14,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41113,DS-8306a2d5-0da1-483b-a228-5fc711830e9a,DISK] 2024-11-16T16:14:14,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-16T16:14:14,979 INFO [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/WALs/303fe01965e8,45907,1731773651719/303fe01965e8%2C45907%2C1731773651719.1731773654915 2024-11-16T16:14:14,980 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187)] 2024-11-16T16:14:14,981 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:14:14,982 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:14,985 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:14,986 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,019 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,040 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T16:14:15,043 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:15,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:15,046 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,049 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T16:14:15,050 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:15,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:15,051 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T16:14:15,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:15,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:15,055 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T16:14:15,058 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:15,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:15,059 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,062 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,063 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,068 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,068 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,071 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T16:14:15,075 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T16:14:15,079 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:14:15,080 INFO [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75133636, jitterRate=0.11957842111587524}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T16:14:15,086 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731773654997Initializing all the Stores at 1731773654999 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773654999Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773655000 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773655000Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773655001 (+1 ms)Cleaning up temporary data from old regions at 1731773655068 (+67 ms)Region opened successfully at 1731773655086 (+18 ms) 2024-11-16T16:14:15,088 INFO [master/303fe01965e8:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T16:14:15,120 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6179bd3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0 2024-11-16T16:14:15,149 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T16:14:15,159 INFO [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T16:14:15,159 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T16:14:15,161 INFO [master/303fe01965e8:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T16:14:15,163 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-16T16:14:15,167 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-16T16:14:15,167 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T16:14:15,190 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T16:14:15,198 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T16:14:15,252 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T16:14:15,256 INFO [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T16:14:15,259 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T16:14:15,328 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T16:14:15,333 INFO [master/303fe01965e8:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T16:14:15,338 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T16:14:15,408 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T16:14:15,411 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T16:14:15,476 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T16:14:15,503 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T16:14:15,524 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T16:14:15,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T16:14:15,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T16:14:15,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,548 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=303fe01965e8,45907,1731773651719, sessionid=0x101446cd3870000, setting cluster-up flag (Was=false) 2024-11-16T16:14:15,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,682 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T16:14:15,685 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,45907,1731773651719 2024-11-16T16:14:15,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:15,735 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T16:14:15,737 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=303fe01965e8,45907,1731773651719 2024-11-16T16:14:15,744 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T16:14:15,813 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T16:14:15,821 INFO [master/303fe01965e8:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T16:14:15,826 INFO [master/303fe01965e8:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T16:14:15,831 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(746): ClusterId : d498f81b-52a7-4718-ba95-d95378acab0e 2024-11-16T16:14:15,834 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T16:14:15,833 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 303fe01965e8,45907,1731773651719 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T16:14:15,842 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T16:14:15,842 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T16:14:15,842 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T16:14:15,842 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/303fe01965e8:0, corePoolSize=5, maxPoolSize=5 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/303fe01965e8:0, corePoolSize=10, maxPoolSize=10 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/303fe01965e8:0, corePoolSize=2, maxPoolSize=2 2024-11-16T16:14:15,843 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:15,845 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731773685845 2024-11-16T16:14:15,847 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T16:14:15,848 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T16:14:15,849 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T16:14:15,849 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T16:14:15,852 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T16:14:15,853 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T16:14:15,853 DEBUG [RS:0;303fe01965e8:37355 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28d99dd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=303fe01965e8/172.17.0.2:0 2024-11-16T16:14:15,853 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T16:14:15,853 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T16:14:15,854 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T16:14:15,856 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:15,855 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:15,857 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T16:14:15,860 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T16:14:15,862 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T16:14:15,862 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T16:14:15,866 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T16:14:15,866 INFO [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T16:14:15,871 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;303fe01965e8:37355 2024-11-16T16:14:15,872 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731773655868,5,FailOnTimeoutGroup] 2024-11-16T16:14:15,873 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731773655872,5,FailOnTimeoutGroup] 2024-11-16T16:14:15,873 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:15,873 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T16:14:15,874 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:15,874 INFO [RS:0;303fe01965e8:37355 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T16:14:15,875 INFO [RS:0;303fe01965e8:37355 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T16:14:15,875 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:15,875 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T16:14:15,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741831_1007 (size=1321) 2024-11-16T16:14:15,878 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(2659): reportForDuty to master=303fe01965e8,45907,1731773651719 with port=37355, startcode=1731773652566 2024-11-16T16:14:15,891 DEBUG [RS:0;303fe01965e8:37355 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T16:14:15,927 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53209, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T16:14:15,935 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45907 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 303fe01965e8,37355,1731773652566 2024-11-16T16:14:15,938 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45907 {}] master.ServerManager(517): Registering regionserver=303fe01965e8,37355,1731773652566 2024-11-16T16:14:15,955 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:14:15,955 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46569 2024-11-16T16:14:15,955 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T16:14:15,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T16:14:15,967 DEBUG [RS:0;303fe01965e8:37355 {}] zookeeper.ZKUtil(111): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/303fe01965e8,37355,1731773652566 2024-11-16T16:14:15,967 WARN [RS:0;303fe01965e8:37355 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T16:14:15,968 INFO [RS:0;303fe01965e8:37355 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T16:14:15,968 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566 2024-11-16T16:14:15,970 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [303fe01965e8,37355,1731773652566] 2024-11-16T16:14:15,995 INFO [RS:0;303fe01965e8:37355 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T16:14:16,015 INFO [RS:0;303fe01965e8:37355 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T16:14:16,021 INFO [RS:0;303fe01965e8:37355 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T16:14:16,021 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,022 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T16:14:16,028 INFO [RS:0;303fe01965e8:37355 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T16:14:16,029 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,030 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/303fe01965e8:0, corePoolSize=2, maxPoolSize=2 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,031 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/303fe01965e8:0, corePoolSize=1, maxPoolSize=1 2024-11-16T16:14:16,032 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3 2024-11-16T16:14:16,032 DEBUG [RS:0;303fe01965e8:37355 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0, corePoolSize=3, maxPoolSize=3 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,035 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37355,1731773652566-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T16:14:16,057 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T16:14:16,059 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,37355,1731773652566-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,060 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,060 INFO [RS:0;303fe01965e8:37355 {}] regionserver.Replication(171): 303fe01965e8,37355,1731773652566 started 2024-11-16T16:14:16,081 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:16,082 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1482): Serving as 303fe01965e8,37355,1731773652566, RpcServer on 303fe01965e8/172.17.0.2:37355, sessionid=0x101446cd3870001 2024-11-16T16:14:16,083 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T16:14:16,083 DEBUG [RS:0;303fe01965e8:37355 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 303fe01965e8,37355,1731773652566 2024-11-16T16:14:16,083 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,37355,1731773652566' 2024-11-16T16:14:16,083 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T16:14:16,084 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T16:14:16,085 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T16:14:16,085 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T16:14:16,085 DEBUG [RS:0;303fe01965e8:37355 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 303fe01965e8,37355,1731773652566 2024-11-16T16:14:16,085 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '303fe01965e8,37355,1731773652566' 2024-11-16T16:14:16,085 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T16:14:16,086 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T16:14:16,086 DEBUG [RS:0;303fe01965e8:37355 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T16:14:16,086 INFO [RS:0;303fe01965e8:37355 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T16:14:16,087 INFO [RS:0;303fe01965e8:37355 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T16:14:16,192 INFO [RS:0;303fe01965e8:37355 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T16:14:16,197 INFO [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C37355%2C1731773652566, suffix=, logDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566, archiveDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/oldWALs, maxLogs=32 2024-11-16T16:14:16,214 DEBUG [RS:0;303fe01965e8:37355 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566/303fe01965e8%2C37355%2C1731773652566.1731773656201, exclude list is [], retry=0 2024-11-16T16:14:16,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41113,DS-8306a2d5-0da1-483b-a228-5fc711830e9a,DISK] 2024-11-16T16:14:16,223 INFO [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566/303fe01965e8%2C37355%2C1731773652566.1731773656201 2024-11-16T16:14:16,223 DEBUG [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187)] 2024-11-16T16:14:16,282 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T16:14:16,284 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:14:16,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741833_1009 (size=32) 2024-11-16T16:14:16,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:16,700 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T16:14:16,704 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T16:14:16,704 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:16,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:16,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T16:14:16,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T16:14:16,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:16,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:16,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T16:14:16,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T16:14:16,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:16,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:16,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T16:14:16,719 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T16:14:16,719 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:16,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:16,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T16:14:16,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740 2024-11-16T16:14:16,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740 2024-11-16T16:14:16,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T16:14:16,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T16:14:16,729 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:14:16,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T16:14:16,736 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:14:16,737 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68604022, jitterRate=0.022279590368270874}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:14:16,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731773656696Initializing all the Stores at 1731773656699 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773656699Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773656699Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773656699Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773656699Cleaning up temporary data from old regions at 1731773656728 (+29 ms)Region opened successfully at 1731773656741 (+13 ms) 2024-11-16T16:14:16,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T16:14:16,741 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T16:14:16,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T16:14:16,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T16:14:16,742 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T16:14:16,743 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T16:14:16,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731773656741Disabling compacts and flushes for region at 1731773656741Disabling writes for close at 1731773656742 (+1 ms)Writing region close event to WAL at 1731773656743 (+1 ms)Closed at 1731773656743 2024-11-16T16:14:16,748 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T16:14:16,748 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T16:14:16,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T16:14:16,763 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T16:14:16,766 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T16:14:16,919 DEBUG [303fe01965e8:45907 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T16:14:16,930 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:16,937 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,37355,1731773652566, state=OPENING 2024-11-16T16:14:16,945 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T16:14:16,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:16,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:14:16,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T16:14:16,957 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T16:14:16,959 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T16:14:16,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:17,141 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T16:14:17,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52477, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T16:14:17,155 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T16:14:17,155 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T16:14:17,156 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T16:14:17,159 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=303fe01965e8%2C37355%2C1731773652566.meta, suffix=.meta, logDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566, archiveDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/oldWALs, maxLogs=32 2024-11-16T16:14:17,174 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566/303fe01965e8%2C37355%2C1731773652566.meta.1731773657161.meta, exclude list is [], retry=0 2024-11-16T16:14:17,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41113,DS-8306a2d5-0da1-483b-a228-5fc711830e9a,DISK] 2024-11-16T16:14:17,180 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/WALs/303fe01965e8,37355,1731773652566/303fe01965e8%2C37355%2C1731773652566.meta.1731773657161.meta 2024-11-16T16:14:17,181 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45187:45187)] 2024-11-16T16:14:17,181 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:14:17,183 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T16:14:17,185 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T16:14:17,189 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T16:14:17,193 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T16:14:17,193 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:17,194 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T16:14:17,194 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T16:14:17,197 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-16T16:14:17,198 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T16:14:17,198 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:17,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:17,199 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-16T16:14:17,200 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T16:14:17,201 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:17,201 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:17,202 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-16T16:14:17,203 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T16:14:17,203 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:17,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:17,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-16T16:14:17,205 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T16:14:17,206 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:17,206 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T16:14:17,207 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T16:14:17,208 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740 2024-11-16T16:14:17,210 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740 2024-11-16T16:14:17,213 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T16:14:17,213 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T16:14:17,214 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:14:17,216 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T16:14:17,218 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61298777, jitterRate=-0.08657704293727875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:14:17,218 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T16:14:17,219 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731773657194Writing region info on filesystem at 1731773657194Initializing all the Stores at 1731773657196 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773657196Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773657196Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773657196Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731773657196Cleaning up temporary data from old regions at 1731773657213 (+17 ms)Running coprocessor post-open hooks at 1731773657218 (+5 ms)Region opened successfully at 1731773657219 (+1 ms) 2024-11-16T16:14:17,226 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731773657133 2024-11-16T16:14:17,239 DEBUG [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T16:14:17,239 INFO [RS_OPEN_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T16:14:17,241 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:17,244 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 303fe01965e8,37355,1731773652566, state=OPEN 2024-11-16T16:14:17,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T16:14:17,367 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T16:14:17,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T16:14:17,368 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T16:14:17,368 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:17,378 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T16:14:17,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=303fe01965e8,37355,1731773652566 in 407 msec 2024-11-16T16:14:17,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T16:14:17,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 625 msec 2024-11-16T16:14:17,387 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T16:14:17,387 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T16:14:17,404 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:17,405 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:17,422 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:17,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:17,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6700 sec 2024-11-16T16:14:17,442 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731773657442, completionTime=-1 2024-11-16T16:14:17,445 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-16T16:14:17,445 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-16T16:14:17,468 INFO [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-16T16:14:17,468 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731773717468 2024-11-16T16:14:17,468 INFO [master/303fe01965e8:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731773777468 2024-11-16T16:14:17,468 INFO [master/303fe01965e8:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-16T16:14:17,470 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,470 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,470 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,472 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-303fe01965e8:45907, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,472 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,473 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,478 DEBUG [master/303fe01965e8:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T16:14:17,500 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.544sec 2024-11-16T16:14:17,501 INFO [master/303fe01965e8:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T16:14:17,502 INFO [master/303fe01965e8:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T16:14:17,503 INFO [master/303fe01965e8:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T16:14:17,504 INFO [master/303fe01965e8:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T16:14:17,504 INFO [master/303fe01965e8:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T16:14:17,504 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T16:14:17,505 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T16:14:17,513 DEBUG [master/303fe01965e8:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T16:14:17,513 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T16:14:17,514 INFO [master/303fe01965e8:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=303fe01965e8,45907,1731773651719-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T16:14:17,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:17,546 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T16:14:17,546 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T16:14:17,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:17,551 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:17,558 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:17,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:17,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:17,563 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:17,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:17,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:17,568 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:17,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:17,572 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:17,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:17,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:17,581 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:17,604 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=303fe01965e8,45907,1731773651719 2024-11-16T16:14:17,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=226, ProcessCount=11, AvailableMemoryMB=4607 2024-11-16T16:14:17,626 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:14:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:14:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680215db 2024-11-16T16:14:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:14:17,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:14:17,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:14:17,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:14:17,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:17,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:14:17,684 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-16T16:14:17,684 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:17,686 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:14:17,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:17,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741835_1011 (size=1061) 2024-11-16T16:14:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:18,106 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:14:18,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741836_1012 (size=53) 2024-11-16T16:14:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:18,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:18,520 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8161ddf0266b9f8b7a8b291995378f22, disabling compactions & flushes 2024-11-16T16:14:18,521 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. after waiting 0 ms 2024-11-16T16:14:18,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,521 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8161ddf0266b9f8b7a8b291995378f22: Waiting for close lock at 1731773658520Disabling compacts and flushes for region at 1731773658520Disabling writes for close at 1731773658521 (+1 ms)Writing region close event to WAL at 1731773658521Closed at 1731773658521 2024-11-16T16:14:18,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:14:18,533 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773658526"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773658526"}]},"ts":"1731773658526"} 2024-11-16T16:14:18,537 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:14:18,539 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:14:18,541 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773658539"}]},"ts":"1731773658539"} 2024-11-16T16:14:18,546 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:14:18,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, ASSIGN}] 2024-11-16T16:14:18,550 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, ASSIGN 2024-11-16T16:14:18,552 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:14:18,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8161ddf0266b9f8b7a8b291995378f22, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:18,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, ASSIGN because future has completed 2024-11-16T16:14:18,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:18,878 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,878 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:14:18,879 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,879 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:18,879 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,880 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,883 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,888 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:18,888 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8161ddf0266b9f8b7a8b291995378f22 columnFamilyName A 2024-11-16T16:14:18,889 DEBUG [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:18,890 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(327): Store=8161ddf0266b9f8b7a8b291995378f22/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:18,891 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,893 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:18,893 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8161ddf0266b9f8b7a8b291995378f22 columnFamilyName B 2024-11-16T16:14:18,894 DEBUG [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:18,894 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(327): Store=8161ddf0266b9f8b7a8b291995378f22/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:18,895 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,897 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:18,897 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8161ddf0266b9f8b7a8b291995378f22 columnFamilyName C 2024-11-16T16:14:18,897 DEBUG [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:18,898 INFO [StoreOpener-8161ddf0266b9f8b7a8b291995378f22-1 {}] regionserver.HStore(327): Store=8161ddf0266b9f8b7a8b291995378f22/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:18,899 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,899 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,900 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,901 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,903 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,903 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,904 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:14:18,906 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,910 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:14:18,911 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8161ddf0266b9f8b7a8b291995378f22; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61858374, jitterRate=-0.07823839783668518}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:14:18,911 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:18,912 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8161ddf0266b9f8b7a8b291995378f22: Running coprocessor pre-open hook at 1731773658880Writing region info on filesystem at 1731773658880Initializing all the Stores at 1731773658882 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773658882Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773658882Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773658883 (+1 ms)Cleaning up temporary data from old regions at 1731773658903 (+20 ms)Running coprocessor post-open hooks at 1731773658911 (+8 ms)Region opened successfully at 1731773658912 (+1 ms) 2024-11-16T16:14:18,914 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., pid=6, masterSystemTime=1731773658869 2024-11-16T16:14:18,918 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,918 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:18,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8161ddf0266b9f8b7a8b291995378f22, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:18,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:14:18,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T16:14:18,932 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 in 213 msec 2024-11-16T16:14:18,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T16:14:18,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, ASSIGN in 383 msec 2024-11-16T16:14:18,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:14:18,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773658937"}]},"ts":"1731773658937"} 2024-11-16T16:14:18,941 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:14:18,943 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:14:18,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2740 sec 2024-11-16T16:14:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T16:14:19,859 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:19,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,871 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bff7409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,874 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2538dc8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,876 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,882 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:14:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:14:19,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@481dbf66 2024-11-16T16:14:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:14:19,890 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:14:19,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@675bb8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,899 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b21050, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,902 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b084ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,906 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,909 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,913 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf9665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,915 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,919 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b3fb7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee049f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,927 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,927 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,928 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,931 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58bfae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,933 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,933 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,933 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,934 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b8551d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,936 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,937 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@517e68dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,938 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,938 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,938 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,939 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,939 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c1b5617, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,941 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,942 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,943 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,943 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:19,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:19,944 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e082570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:19,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:19,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cc1ac04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:19,951 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,951 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,951 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,951 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,952 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,953 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,954 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,955 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60694, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,956 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,959 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,960 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,962 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,978 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,982 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:19,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-16T16:14:19,984 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:19,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T16:14:19,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:19,985 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:19,986 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:19,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:19,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:19,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:14:20,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:14:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:20,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:20,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T16:14:20,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/0704695b0b99428489e033a87276a05a is 50, key is test_row_0/A:col10/1731773660034/Put/seqid=0 2024-11-16T16:14:20,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T16:14:20,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773720136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741837_1013 (size=14341) 2024-11-16T16:14:20,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/0704695b0b99428489e033a87276a05a 2024-11-16T16:14:20,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:20,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773720145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773720139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773720152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773720159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8036fdc8f423432a9311f88f884d36cc is 50, key is test_row_0/B:col10/1731773660034/Put/seqid=0 2024-11-16T16:14:20,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741838_1014 (size=12001) 2024-11-16T16:14:20,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8036fdc8f423432a9311f88f884d36cc 2024-11-16T16:14:20,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T16:14:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773720306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773720307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773720308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773720308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773720315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T16:14:20,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:20,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/bc7c478a46e548d188612b248e47540e is 50, key is test_row_0/C:col10/1731773660034/Put/seqid=0 2024-11-16T16:14:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741839_1015 (size=12001) 2024-11-16T16:14:20,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/bc7c478a46e548d188612b248e47540e 2024-11-16T16:14:20,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/0704695b0b99428489e033a87276a05a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a 2024-11-16T16:14:20,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a, entries=200, sequenceid=13, filesize=14.0 K 2024-11-16T16:14:20,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8036fdc8f423432a9311f88f884d36cc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc 2024-11-16T16:14:20,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T16:14:20,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/bc7c478a46e548d188612b248e47540e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e 2024-11-16T16:14:20,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T16:14:20,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8161ddf0266b9f8b7a8b291995378f22 in 413ms, sequenceid=13, compaction requested=false 2024-11-16T16:14:20,453 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T16:14:20,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:20,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-16T16:14:20,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,501 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:14:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:20,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:20,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:20,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/44f1e01a7fd246a3ac6e955557ef640f is 50, key is test_row_0/A:col10/1731773660128/Put/seqid=0 2024-11-16T16:14:20,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741840_1016 (size=12001) 2024-11-16T16:14:20,564 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/44f1e01a7fd246a3ac6e955557ef640f 2024-11-16T16:14:20,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8e93fa080ee540a4b9f57b96bed6543b is 50, key is test_row_0/B:col10/1731773660128/Put/seqid=0 2024-11-16T16:14:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773720563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773720590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773720599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T16:14:20,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773720608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773720605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,629 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741841_1017 (size=12001) 2024-11-16T16:14:20,633 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8e93fa080ee540a4b9f57b96bed6543b 2024-11-16T16:14:20,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/37226bfdda114828a4686d4fb7b3fa90 is 50, key is test_row_0/C:col10/1731773660128/Put/seqid=0 2024-11-16T16:14:20,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741842_1018 (size=12001) 2024-11-16T16:14:20,667 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/37226bfdda114828a4686d4fb7b3fa90 2024-11-16T16:14:20,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/44f1e01a7fd246a3ac6e955557ef640f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f 2024-11-16T16:14:20,709 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:14:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8e93fa080ee540a4b9f57b96bed6543b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b 2024-11-16T16:14:20,726 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:14:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/37226bfdda114828a4686d4fb7b3fa90 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90 2024-11-16T16:14:20,749 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:14:20,751 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8161ddf0266b9f8b7a8b291995378f22 in 250ms, sequenceid=33, compaction requested=false 2024-11-16T16:14:20,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:20,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-16T16:14:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-16T16:14:20,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T16:14:20,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:20,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 768 msec 2024-11-16T16:14:20,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T16:14:20,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:20,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:20,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:20,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:20,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 790 msec 2024-11-16T16:14:20,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6c4fc922a040449ba883f89ee6aaf5e3 is 50, key is test_row_0/A:col10/1731773660754/Put/seqid=0 2024-11-16T16:14:20,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741843_1019 (size=12001) 2024-11-16T16:14:20,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6c4fc922a040449ba883f89ee6aaf5e3 2024-11-16T16:14:20,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/da66575a43174b77aad2557ce2c4647c is 50, key is test_row_0/B:col10/1731773660754/Put/seqid=0 2024-11-16T16:14:20,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741844_1020 (size=12001) 2024-11-16T16:14:20,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/da66575a43174b77aad2557ce2c4647c 2024-11-16T16:14:20,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/db40b69b9c8048caa887c0af03f58222 is 50, key is test_row_0/C:col10/1731773660754/Put/seqid=0 2024-11-16T16:14:20,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741845_1021 (size=12001) 2024-11-16T16:14:20,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/db40b69b9c8048caa887c0af03f58222 2024-11-16T16:14:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773720873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773720866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773720874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773720882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,901 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6c4fc922a040449ba883f89ee6aaf5e3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3 2024-11-16T16:14:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:20,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773720883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:20,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:20,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T16:14:20,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/da66575a43174b77aad2557ce2c4647c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c 2024-11-16T16:14:20,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T16:14:20,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/db40b69b9c8048caa887c0af03f58222 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222 2024-11-16T16:14:20,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T16:14:20,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8161ddf0266b9f8b7a8b291995378f22 in 195ms, sequenceid=49, compaction requested=true 2024-11-16T16:14:20,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:20,964 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:20,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:20,966 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:20,967 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:20,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:20,970 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:20,972 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:20,973 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,973 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=35.2 K 2024-11-16T16:14:20,975 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8036fdc8f423432a9311f88f884d36cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773660024 2024-11-16T16:14:20,979 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e93fa080ee540a4b9f57b96bed6543b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773660111 2024-11-16T16:14:20,979 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:20,979 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:20,979 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:20,979 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.4 K 2024-11-16T16:14:20,980 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting da66575a43174b77aad2557ce2c4647c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773660596 2024-11-16T16:14:20,981 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0704695b0b99428489e033a87276a05a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773660024 2024-11-16T16:14:20,983 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44f1e01a7fd246a3ac6e955557ef640f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773660111 2024-11-16T16:14:20,987 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4fc922a040449ba883f89ee6aaf5e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773660596 2024-11-16T16:14:21,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:21,018 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:21,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/1a8e09b9978e47bda389f0207f47d885 is 50, key is test_row_0/A:col10/1731773660833/Put/seqid=0 2024-11-16T16:14:21,046 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:21,048 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b1eea18aaa64495cabaf68702d431036 is 50, key is test_row_0/B:col10/1731773660754/Put/seqid=0 2024-11-16T16:14:21,052 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#11 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:21,054 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6aecfffa99fb4fd399009ab463586886 is 50, key is test_row_0/A:col10/1731773660754/Put/seqid=0 2024-11-16T16:14:21,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741846_1022 (size=14341) 2024-11-16T16:14:21,085 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/1a8e09b9978e47bda389f0207f47d885 2024-11-16T16:14:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773721068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773721069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773721073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773721086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,106 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773721086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741847_1023 (size=12104) 2024-11-16T16:14:21,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-16T16:14:21,136 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:21,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:21,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-16T16:14:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:21,142 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:21,143 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:21,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:21,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cb2d77051ce04bb691b389469de786e7 is 50, key is test_row_0/B:col10/1731773660833/Put/seqid=0 2024-11-16T16:14:21,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741848_1024 (size=12104) 2024-11-16T16:14:21,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741849_1025 (size=12001) 2024-11-16T16:14:21,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cb2d77051ce04bb691b389469de786e7 2024-11-16T16:14:21,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773721196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773721196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773721198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6860afd139214baa957b90f5e61d48fc is 50, key is test_row_0/C:col10/1731773660833/Put/seqid=0 2024-11-16T16:14:21,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773721217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,223 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773721217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741850_1026 (size=12001) 2024-11-16T16:14:21,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6860afd139214baa957b90f5e61d48fc 2024-11-16T16:14:21,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:21,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/1a8e09b9978e47bda389f0207f47d885 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885 2024-11-16T16:14:21,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885, entries=200, sequenceid=66, filesize=14.0 K 2024-11-16T16:14:21,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cb2d77051ce04bb691b389469de786e7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7 2024-11-16T16:14:21,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7, entries=150, sequenceid=66, filesize=11.7 K 2024-11-16T16:14:21,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6860afd139214baa957b90f5e61d48fc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc 2024-11-16T16:14:21,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:21,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc, entries=150, sequenceid=66, filesize=11.7 K 2024-11-16T16:14:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8161ddf0266b9f8b7a8b291995378f22 in 285ms, sequenceid=66, compaction requested=true 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:21,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-16T16:14:21,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:21,409 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:21,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:21,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9920660515d64d34bc60ab4e7f3e06e1 is 50, key is test_row_0/A:col10/1731773661406/Put/seqid=0 2024-11-16T16:14:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:21,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:21,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741851_1027 (size=14341) 2024-11-16T16:14:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:21,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9920660515d64d34bc60ab4e7f3e06e1 2024-11-16T16:14:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4526c09f2b9f42fe813737046b9b492c is 50, key is test_row_0/B:col10/1731773661406/Put/seqid=0 2024-11-16T16:14:21,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773721471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773721473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773721477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773721482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773721483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741852_1028 (size=12001) 2024-11-16T16:14:21,523 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b1eea18aaa64495cabaf68702d431036 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b1eea18aaa64495cabaf68702d431036 2024-11-16T16:14:21,546 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into b1eea18aaa64495cabaf68702d431036(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:21,546 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:21,549 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773660966; duration=0sec 2024-11-16T16:14:21,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-16T16:14:21,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:21,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-16T16:14:21,551 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T16:14:21,551 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T16:14:21,551 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. because compaction request was cancelled 2024-11-16T16:14:21,552 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:21,552 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:21,557 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:21,557 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:21,557 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,557 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=46.9 K 2024-11-16T16:14:21,559 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7c478a46e548d188612b248e47540e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773660024 2024-11-16T16:14:21,560 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 37226bfdda114828a4686d4fb7b3fa90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773660111 2024-11-16T16:14:21,560 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting db40b69b9c8048caa887c0af03f58222, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773660596 2024-11-16T16:14:21,561 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6860afd139214baa957b90f5e61d48fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731773660833 2024-11-16T16:14:21,582 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6aecfffa99fb4fd399009ab463586886 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6aecfffa99fb4fd399009ab463586886 2024-11-16T16:14:21,600 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 6aecfffa99fb4fd399009ab463586886(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:21,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:21,600 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773660957; duration=0sec 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. because compaction request was cancelled 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T16:14:21,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773721597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T16:14:21,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T16:14:21,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. because compaction request was cancelled 2024-11-16T16:14:21,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773721599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,604 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#16 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,605 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4c0548b6fdd84d7dbdeaed04f43e4a4c is 50, key is test_row_0/C:col10/1731773660833/Put/seqid=0 2024-11-16T16:14:21,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773721599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:21,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773721606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773721607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741853_1029 (size=12139) 2024-11-16T16:14:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:21,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:21,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773721816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773721816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773721826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773721827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:21,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773721827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:21,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:21,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:21,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4526c09f2b9f42fe813737046b9b492c 2024-11-16T16:14:21,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e82bfbe829ca41b2a90b27e35daf778e is 50, key is test_row_0/C:col10/1731773661406/Put/seqid=0 2024-11-16T16:14:21,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:21,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:21,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:21,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741854_1030 (size=12001) 2024-11-16T16:14:21,956 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e82bfbe829ca41b2a90b27e35daf778e 2024-11-16T16:14:21,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9920660515d64d34bc60ab4e7f3e06e1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1 2024-11-16T16:14:21,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1, entries=200, sequenceid=82, filesize=14.0 K 2024-11-16T16:14:21,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4526c09f2b9f42fe813737046b9b492c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c 2024-11-16T16:14:21,987 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T16:14:21,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T16:14:22,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e82bfbe829ca41b2a90b27e35daf778e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e 2024-11-16T16:14:22,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T16:14:22,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8161ddf0266b9f8b7a8b291995378f22 in 609ms, sequenceid=82, compaction requested=true 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:22,019 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:14:22,022 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:22,022 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:22,022 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:22,022 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6aecfffa99fb4fd399009ab463586886, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=39.8 K 2024-11-16T16:14:22,023 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aecfffa99fb4fd399009ab463586886, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773660596 2024-11-16T16:14:22,024 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a8e09b9978e47bda389f0207f47d885, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731773660833 2024-11-16T16:14:22,025 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9920660515d64d34bc60ab4e7f3e06e1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773661067 2024-11-16T16:14:22,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4c0548b6fdd84d7dbdeaed04f43e4a4c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4c0548b6fdd84d7dbdeaed04f43e4a4c 2024-11-16T16:14:22,055 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 4c0548b6fdd84d7dbdeaed04f43e4a4c(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:22,055 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:22,055 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=12, startTime=1731773661303; duration=0sec 2024-11-16T16:14:22,055 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:14:22,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:22,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:22,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:22,058 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:22,058 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:22,058 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:22,058 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b1eea18aaa64495cabaf68702d431036, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=35.3 K 2024-11-16T16:14:22,058 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#18 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:22,059 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/cb9509c7689146dd8380ab1b54e9b116 is 50, key is test_row_0/A:col10/1731773661406/Put/seqid=0 2024-11-16T16:14:22,059 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b1eea18aaa64495cabaf68702d431036, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773660596 2024-11-16T16:14:22,061 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2d77051ce04bb691b389469de786e7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731773660833 2024-11-16T16:14:22,062 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4526c09f2b9f42fe813737046b9b492c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773661067 2024-11-16T16:14:22,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-16T16:14:22,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:22,087 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:22,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:22,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741855_1031 (size=12207) 2024-11-16T16:14:22,105 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#19 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:22,106 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cbd9c5c110614317a77211760e8b41d2 is 50, key is test_row_0/B:col10/1731773661406/Put/seqid=0 2024-11-16T16:14:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/b3588ba2b5ad48198ab7cef10f8cae70 is 50, key is test_row_0/A:col10/1731773661476/Put/seqid=0 2024-11-16T16:14:22,119 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/cb9509c7689146dd8380ab1b54e9b116 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/cb9509c7689146dd8380ab1b54e9b116 2024-11-16T16:14:22,132 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into cb9509c7689146dd8380ab1b54e9b116(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:22,134 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:22,134 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773662019; duration=0sec 2024-11-16T16:14:22,134 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:22,134 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:22,134 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-16T16:14:22,137 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-16T16:14:22,137 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-16T16:14:22,137 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. because compaction request was cancelled 2024-11-16T16:14:22,137 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:22,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:22,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T16:14:22,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741857_1033 (size=12001) 2024-11-16T16:14:22,155 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T16:14:22,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741856_1032 (size=12207) 2024-11-16T16:14:22,162 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T16:14:22,162 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T16:14:22,163 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T16:14:22,163 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T16:14:22,165 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:14:22,165 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T16:14:22,177 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cbd9c5c110614317a77211760e8b41d2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cbd9c5c110614317a77211760e8b41d2 2024-11-16T16:14:22,188 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into cbd9c5c110614317a77211760e8b41d2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:22,188 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:22,188 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773662019; duration=0sec 2024-11-16T16:14:22,189 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:22,189 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773722190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773722193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,215 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773722201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,216 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773722202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773722201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:22,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773722305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773722316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773722325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773722326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773722336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773722515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773722526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773722536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773722536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773722545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,549 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,613 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/b3588ba2b5ad48198ab7cef10f8cae70 2024-11-16T16:14:22,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 is 50, key is test_row_0/B:col10/1731773661476/Put/seqid=0 2024-11-16T16:14:22,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741858_1034 (size=12001) 2024-11-16T16:14:22,721 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:14:22,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773722825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773722835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773722845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773722845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:22,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:22,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773722855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:22,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,059 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 2024-11-16T16:14:23,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/50df3a42484f41e48cc30a87966da53a is 50, key is test_row_0/C:col10/1731773661476/Put/seqid=0 2024-11-16T16:14:23,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741859_1035 (size=12001) 2024-11-16T16:14:23,115 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/50df3a42484f41e48cc30a87966da53a 2024-11-16T16:14:23,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/b3588ba2b5ad48198ab7cef10f8cae70 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70 2024-11-16T16:14:23,144 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T16:14:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 2024-11-16T16:14:23,163 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T16:14:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/50df3a42484f41e48cc30a87966da53a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a 2024-11-16T16:14:23,184 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T16:14:23,187 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for 8161ddf0266b9f8b7a8b291995378f22 in 1099ms, sequenceid=102, compaction requested=true 2024-11-16T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-16T16:14:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-16T16:14:23,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-16T16:14:23,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0470 sec 2024-11-16T16:14:23,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.0620 sec 2024-11-16T16:14:23,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-16T16:14:23,286 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:23,287 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-16T16:14:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:23,290 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:23,293 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:23,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:23,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T16:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:23,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:23,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:23,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a1f68e8fd6704579a1aa12f252b68128 is 50, key is test_row_0/A:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:23,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773723374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773723376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,383 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773723377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773723379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741860_1036 (size=14341) 2024-11-16T16:14:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773723382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,387 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a1f68e8fd6704579a1aa12f252b68128 2024-11-16T16:14:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:23,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/39697d7e1973421898f6e619a1e8068b is 50, key is test_row_0/B:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741861_1037 (size=12001) 2024-11-16T16:14:23,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T16:14:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773723486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773723486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,497 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773723496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773723496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773723496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T16:14:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:23,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773723705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773723706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773723706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773723706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773723706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:23,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:23,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T16:14:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:23,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:23,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/39697d7e1973421898f6e619a1e8068b 2024-11-16T16:14:23,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/da207e32d5444fbab2eedfc1873789f3 is 50, key is test_row_0/C:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:23,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741862_1038 (size=12001) 2024-11-16T16:14:23,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/da207e32d5444fbab2eedfc1873789f3 2024-11-16T16:14:23,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a1f68e8fd6704579a1aa12f252b68128 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128 2024-11-16T16:14:23,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128, entries=200, sequenceid=122, filesize=14.0 K 2024-11-16T16:14:23,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/39697d7e1973421898f6e619a1e8068b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b 2024-11-16T16:14:23,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b, entries=150, sequenceid=122, filesize=11.7 K 2024-11-16T16:14:23,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/da207e32d5444fbab2eedfc1873789f3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3 2024-11-16T16:14:23,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3, entries=150, sequenceid=122, filesize=11.7 K 2024-11-16T16:14:23,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 8161ddf0266b9f8b7a8b291995378f22 in 560ms, sequenceid=122, compaction requested=true 2024-11-16T16:14:23,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:23,909 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:23,909 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:23,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:23,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-16T16:14:23,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,911 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:23,911 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T16:14:23,911 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:23,911 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:23,911 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cbd9c5c110614317a77211760e8b41d2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=35.4 K 2024-11-16T16:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:23,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:23,912 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:23,913 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:23,913 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:23,913 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/cb9509c7689146dd8380ab1b54e9b116, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.6 K 2024-11-16T16:14:23,913 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd9c5c110614317a77211760e8b41d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773661067 2024-11-16T16:14:23,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:23,916 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9509c7689146dd8380ab1b54e9b116, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773661067 2024-11-16T16:14:23,918 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a0cfcbe9d232495ca3bcc53a65fe1bb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773661470 2024-11-16T16:14:23,918 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3588ba2b5ad48198ab7cef10f8cae70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773661470 2024-11-16T16:14:23,919 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 39697d7e1973421898f6e619a1e8068b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:23,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ad6fc873404c4d40ac8417f181cbe829 is 50, key is test_row_0/A:col10/1731773663377/Put/seqid=0 2024-11-16T16:14:23,923 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1f68e8fd6704579a1aa12f252b68128, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:23,940 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#27 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:23,941 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/6e675ccd57ab491ba3b05cd9bafbfaff is 50, key is test_row_0/B:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:23,946 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#28 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:23,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741863_1039 (size=12151) 2024-11-16T16:14:23,946 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/e5778a7fe32440908e22cd3627335be8 is 50, key is test_row_0/A:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:23,948 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ad6fc873404c4d40ac8417f181cbe829 2024-11-16T16:14:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8088388a076448c2ae351c109038e7f5 is 50, key is test_row_0/B:col10/1731773663377/Put/seqid=0 2024-11-16T16:14:24,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741865_1041 (size=12309) 2024-11-16T16:14:24,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/e5778a7fe32440908e22cd3627335be8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/e5778a7fe32440908e22cd3627335be8 2024-11-16T16:14:24,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741864_1040 (size=12309) 2024-11-16T16:14:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:24,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:24,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741866_1042 (size=12151) 2024-11-16T16:14:24,029 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into e5778a7fe32440908e22cd3627335be8(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:24,029 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:24,029 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773663909; duration=0sec 2024-11-16T16:14:24,029 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:24,030 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:24,030 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:24,032 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/6e675ccd57ab491ba3b05cd9bafbfaff as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6e675ccd57ab491ba3b05cd9bafbfaff 2024-11-16T16:14:24,035 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48142 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:24,038 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:24,038 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:24,039 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4c0548b6fdd84d7dbdeaed04f43e4a4c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=47.0 K 2024-11-16T16:14:24,042 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c0548b6fdd84d7dbdeaed04f43e4a4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731773660833 2024-11-16T16:14:24,042 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 6e675ccd57ab491ba3b05cd9bafbfaff(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:24,043 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:24,043 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773663909; duration=0sec 2024-11-16T16:14:24,043 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e82bfbe829ca41b2a90b27e35daf778e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773661067 2024-11-16T16:14:24,043 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:24,043 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:24,043 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50df3a42484f41e48cc30a87966da53a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773661470 2024-11-16T16:14:24,044 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting da207e32d5444fbab2eedfc1873789f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:24,069 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:24,070 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/66aa639e003f4a9cb4fe49291aed1a18 is 50, key is test_row_0/C:col10/1731773663346/Put/seqid=0 2024-11-16T16:14:24,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773724055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773724069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,073 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773724069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,078 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773724072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773724072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741867_1043 (size=12275) 2024-11-16T16:14:24,116 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/66aa639e003f4a9cb4fe49291aed1a18 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/66aa639e003f4a9cb4fe49291aed1a18 2024-11-16T16:14:24,129 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 66aa639e003f4a9cb4fe49291aed1a18(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:24,130 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:24,130 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=12, startTime=1731773663909; duration=0sec 2024-11-16T16:14:24,130 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:24,130 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:24,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773724175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773724185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773724185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773724186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773724186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773724384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,389 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773724395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773724395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773724395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773724395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:24,425 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8088388a076448c2ae351c109038e7f5 2024-11-16T16:14:24,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c419ef1a8fbc4f82a7f7746e470713ba is 50, key is test_row_0/C:col10/1731773663377/Put/seqid=0 2024-11-16T16:14:24,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741868_1044 (size=12151) 2024-11-16T16:14:24,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773724695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773724705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773724706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773724706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773724706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:24,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:24,846 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c419ef1a8fbc4f82a7f7746e470713ba 2024-11-16T16:14:24,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ad6fc873404c4d40ac8417f181cbe829 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829 2024-11-16T16:14:24,872 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829, entries=150, sequenceid=136, filesize=11.9 K 2024-11-16T16:14:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8088388a076448c2ae351c109038e7f5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5 2024-11-16T16:14:24,882 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5, entries=150, sequenceid=136, filesize=11.9 K 2024-11-16T16:14:24,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c419ef1a8fbc4f82a7f7746e470713ba as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba 2024-11-16T16:14:24,892 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba, entries=150, sequenceid=136, filesize=11.9 K 2024-11-16T16:14:24,895 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 8161ddf0266b9f8b7a8b291995378f22 in 984ms, sequenceid=136, compaction requested=false 2024-11-16T16:14:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-16T16:14:24,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-16T16:14:24,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-16T16:14:24,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-11-16T16:14:24,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6190 sec 2024-11-16T16:14:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:25,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:25,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/7979293280814b91879c276ebffc8449 is 50, key is test_row_0/A:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773725229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773725229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773725233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773725238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773725238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741869_1045 (size=14541) 2024-11-16T16:14:25,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/7979293280814b91879c276ebffc8449 2024-11-16T16:14:25,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ad96960d7fa641d7998c97ff933adfc9 is 50, key is test_row_0/B:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741870_1046 (size=12151) 2024-11-16T16:14:25,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773725345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773725345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773725345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773725355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773725355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-16T16:14:25,445 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:25,446 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-16T16:14:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:25,448 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:25,450 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:25,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:25,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773725556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773725556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773725559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773725565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773725566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T16:14:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ad96960d7fa641d7998c97ff933adfc9 2024-11-16T16:14:25,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45f3ef78d7bc48e8a3297e424e2d2db4 is 50, key is test_row_0/C:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741871_1047 (size=12151) 2024-11-16T16:14:25,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45f3ef78d7bc48e8a3297e424e2d2db4 2024-11-16T16:14:25,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/7979293280814b91879c276ebffc8449 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449 2024-11-16T16:14:25,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449, entries=200, sequenceid=158, filesize=14.2 K 2024-11-16T16:14:25,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ad96960d7fa641d7998c97ff933adfc9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9 2024-11-16T16:14:25,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T16:14:25,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45f3ef78d7bc48e8a3297e424e2d2db4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4 2024-11-16T16:14:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:25,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T16:14:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:25,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4, entries=150, sequenceid=158, filesize=11.9 K 2024-11-16T16:14:25,773 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8161ddf0266b9f8b7a8b291995378f22 in 565ms, sequenceid=158, compaction requested=true 2024-11-16T16:14:25,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:25,774 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:25,774 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:25,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:25,775 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:25,775 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:25,776 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:25,776 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:25,776 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,776 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,776 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6e675ccd57ab491ba3b05cd9bafbfaff, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=35.8 K 2024-11-16T16:14:25,776 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/e5778a7fe32440908e22cd3627335be8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=38.1 K 2024-11-16T16:14:25,776 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5778a7fe32440908e22cd3627335be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:25,776 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e675ccd57ab491ba3b05cd9bafbfaff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:25,777 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad6fc873404c4d40ac8417f181cbe829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731773663363 2024-11-16T16:14:25,777 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8088388a076448c2ae351c109038e7f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731773663363 2024-11-16T16:14:25,777 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7979293280814b91879c276ebffc8449, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664053 2024-11-16T16:14:25,777 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ad96960d7fa641d7998c97ff933adfc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664068 2024-11-16T16:14:25,792 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#35 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:25,792 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/4ce1061f73dc421188c4e0c69f704179 is 50, key is test_row_0/A:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,795 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#36 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:25,795 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cfe7f912645840f38268abc9869f5c28 is 50, key is test_row_0/B:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741872_1048 (size=12561) 2024-11-16T16:14:25,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741873_1049 (size=12561) 2024-11-16T16:14:25,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/4ce1061f73dc421188c4e0c69f704179 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/4ce1061f73dc421188c4e0c69f704179 2024-11-16T16:14:25,826 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/cfe7f912645840f38268abc9869f5c28 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cfe7f912645840f38268abc9869f5c28 2024-11-16T16:14:25,834 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 4ce1061f73dc421188c4e0c69f704179(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:25,834 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:25,834 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773665774; duration=0sec 2024-11-16T16:14:25,834 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:25,834 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:25,834 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:25,836 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36577 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:25,836 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:25,836 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,837 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/66aa639e003f4a9cb4fe49291aed1a18, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=35.7 K 2024-11-16T16:14:25,839 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into cfe7f912645840f38268abc9869f5c28(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:25,839 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:25,839 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773665774; duration=0sec 2024-11-16T16:14:25,839 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:25,840 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:25,840 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66aa639e003f4a9cb4fe49291aed1a18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731773662185 2024-11-16T16:14:25,841 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c419ef1a8fbc4f82a7f7746e470713ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731773663363 2024-11-16T16:14:25,841 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45f3ef78d7bc48e8a3297e424e2d2db4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664068 2024-11-16T16:14:25,855 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#37 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:25,855 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/96b28b25dc7e499c84c3a61446207b07 is 50, key is test_row_0/C:col10/1731773664068/Put/seqid=0 2024-11-16T16:14:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:25,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:25,868 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:25,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741874_1050 (size=12527) 2024-11-16T16:14:25,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/13ae1defd3b84427872f3b54750936f9 is 50, key is test_row_0/A:col10/1731773665866/Put/seqid=0 2024-11-16T16:14:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773725898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773725898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773725900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773725904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773725904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:25,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741875_1051 (size=12151) 2024-11-16T16:14:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:25,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:25,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/13ae1defd3b84427872f3b54750936f9 2024-11-16T16:14:25,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T16:14:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:25,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:25,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/6703975dc9c04b599c56cf01644b3834 is 50, key is test_row_0/B:col10/1731773665866/Put/seqid=0 2024-11-16T16:14:25,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741876_1052 (size=12151) 2024-11-16T16:14:25,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/6703975dc9c04b599c56cf01644b3834 2024-11-16T16:14:25,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/674d0047bd1c4ce49c84a9ede6c76988 is 50, key is test_row_0/C:col10/1731773665866/Put/seqid=0 2024-11-16T16:14:25,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741877_1053 (size=12151) 2024-11-16T16:14:25,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/674d0047bd1c4ce49c84a9ede6c76988 2024-11-16T16:14:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/13ae1defd3b84427872f3b54750936f9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9 2024-11-16T16:14:26,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T16:14:26,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773726015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773726016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/6703975dc9c04b599c56cf01644b3834 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834 2024-11-16T16:14:26,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773726025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T16:14:26,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/674d0047bd1c4ce49c84a9ede6c76988 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988 2024-11-16T16:14:26,055 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988, entries=150, sequenceid=176, filesize=11.9 K 2024-11-16T16:14:26,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8161ddf0266b9f8b7a8b291995378f22 in 190ms, sequenceid=176, compaction requested=false 2024-11-16T16:14:26,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:26,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-16T16:14:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:26,078 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:14:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:26,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:26,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/90c77010d0b74a0588df54e4ea8e1fbe is 50, key is test_row_0/A:col10/1731773665900/Put/seqid=0 2024-11-16T16:14:26,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741878_1054 (size=12151) 2024-11-16T16:14:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:26,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:26,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773726251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773726253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773726255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,262 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773726255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773726257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,287 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/96b28b25dc7e499c84c3a61446207b07 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/96b28b25dc7e499c84c3a61446207b07 2024-11-16T16:14:26,298 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 96b28b25dc7e499c84c3a61446207b07(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:26,298 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:26,298 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773665774; duration=0sec 2024-11-16T16:14:26,298 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:26,298 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:26,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773726365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773726366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773726366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773726375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773726375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,505 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/90c77010d0b74a0588df54e4ea8e1fbe 2024-11-16T16:14:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b935f19de819439c8b413953ba0537e7 is 50, key is test_row_0/B:col10/1731773665900/Put/seqid=0 2024-11-16T16:14:26,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741879_1055 (size=12151) 2024-11-16T16:14:26,531 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b935f19de819439c8b413953ba0537e7 2024-11-16T16:14:26,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/92051a003a054c2392f7e421e1f71c0a is 50, key is test_row_0/C:col10/1731773665900/Put/seqid=0 2024-11-16T16:14:26,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741880_1056 (size=12151) 2024-11-16T16:14:26,568 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/92051a003a054c2392f7e421e1f71c0a 2024-11-16T16:14:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773726575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773726575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773726575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/90c77010d0b74a0588df54e4ea8e1fbe as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe 2024-11-16T16:14:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773726585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,587 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773726585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,593 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe, entries=150, sequenceid=193, filesize=11.9 K 2024-11-16T16:14:26,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b935f19de819439c8b413953ba0537e7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7 2024-11-16T16:14:26,604 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7, entries=150, sequenceid=193, filesize=11.9 K 2024-11-16T16:14:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/92051a003a054c2392f7e421e1f71c0a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a 2024-11-16T16:14:26,614 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-16T16:14:26,615 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8161ddf0266b9f8b7a8b291995378f22 in 537ms, sequenceid=193, compaction requested=true 2024-11-16T16:14:26,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-16T16:14:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-16T16:14:26,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-16T16:14:26,623 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1680 sec 2024-11-16T16:14:26,627 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.1780 sec 2024-11-16T16:14:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:26,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:26,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:26,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:26,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/bbb5bcfdfc564f049c1df5a2a10d1f3d is 50, key is test_row_0/A:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:26,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741881_1057 (size=14541) 2024-11-16T16:14:26,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/bbb5bcfdfc564f049c1df5a2a10d1f3d 2024-11-16T16:14:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773726931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773726928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773726940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773726944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773726944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:26,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:26,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:26,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/bad6c420e0b74101bddb3a9fc8aa30b3 is 50, key is test_row_0/B:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:26,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741882_1058 (size=12151) 2024-11-16T16:14:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773727055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773727055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773727055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773727056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773727056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,062 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773727265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773727265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773727265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773727265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773727265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/bad6c420e0b74101bddb3a9fc8aa30b3 2024-11-16T16:14:27,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/49f5a22d9b8c4086b74148d4f9ee86a4 is 50, key is test_row_0/C:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:27,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741883_1059 (size=12151) 2024-11-16T16:14:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773727575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773727575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773727575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773727575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773727585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:27,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-16T16:14:27,597 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:27,597 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-16T16:14:27,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T16:14:27,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:27,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T16:14:27,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T16:14:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:27,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/49f5a22d9b8c4086b74148d4f9ee86a4 2024-11-16T16:14:27,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:27,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:27,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/bbb5bcfdfc564f049c1df5a2a10d1f3d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d 2024-11-16T16:14:27,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d, entries=200, sequenceid=210, filesize=14.2 K 2024-11-16T16:14:27,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/bad6c420e0b74101bddb3a9fc8aa30b3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3 2024-11-16T16:14:27,821 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T16:14:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/49f5a22d9b8c4086b74148d4f9ee86a4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4 2024-11-16T16:14:27,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4, entries=150, sequenceid=210, filesize=11.9 K 2024-11-16T16:14:27,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8161ddf0266b9f8b7a8b291995378f22 in 947ms, sequenceid=210, compaction requested=true 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:27,834 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:27,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:27,834 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:27,836 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:27,836 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:27,837 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:27,837 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:27,837 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,837 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cfe7f912645840f38268abc9869f5c28, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=47.9 K 2024-11-16T16:14:27,837 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,837 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/4ce1061f73dc421188c4e0c69f704179, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=50.2 K 2024-11-16T16:14:27,838 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cfe7f912645840f38268abc9869f5c28, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664068 2024-11-16T16:14:27,838 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce1061f73dc421188c4e0c69f704179, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664068 2024-11-16T16:14:27,839 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6703975dc9c04b599c56cf01644b3834, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731773665231 2024-11-16T16:14:27,839 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13ae1defd3b84427872f3b54750936f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731773665231 2024-11-16T16:14:27,839 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90c77010d0b74a0588df54e4ea8e1fbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731773665897 2024-11-16T16:14:27,839 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b935f19de819439c8b413953ba0537e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731773665897 2024-11-16T16:14:27,840 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbb5bcfdfc564f049c1df5a2a10d1f3d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666242 2024-11-16T16:14:27,840 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting bad6c420e0b74101bddb3a9fc8aa30b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666250 2024-11-16T16:14:27,853 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#47 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:27,853 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e096ab71e5424918adc219dbdd12dda5 is 50, key is test_row_0/B:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:27,857 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:27,858 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/548fbda844f24e8da8329b22234177b5 is 50, key is test_row_0/A:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741885_1061 (size=12697) 2024-11-16T16:14:27,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741884_1060 (size=12697) 2024-11-16T16:14:27,918 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/548fbda844f24e8da8329b22234177b5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/548fbda844f24e8da8329b22234177b5 2024-11-16T16:14:27,927 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 548fbda844f24e8da8329b22234177b5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:27,927 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:27,927 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=12, startTime=1731773667834; duration=0sec 2024-11-16T16:14:27,927 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:27,927 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:27,928 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:27,929 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:27,930 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:27,930 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,930 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/96b28b25dc7e499c84c3a61446207b07, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=47.8 K 2024-11-16T16:14:27,931 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b28b25dc7e499c84c3a61446207b07, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731773664068 2024-11-16T16:14:27,931 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 674d0047bd1c4ce49c84a9ede6c76988, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731773665231 2024-11-16T16:14:27,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92051a003a054c2392f7e421e1f71c0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731773665897 2024-11-16T16:14:27,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f5a22d9b8c4086b74148d4f9ee86a4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666250 2024-11-16T16:14:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T16:14:27,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-16T16:14:27,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:27,956 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:27,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:27,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:27,959 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:27,960 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/514e86b528374c9bb9f9a55832e024b1 is 50, key is test_row_0/C:col10/1731773666255/Put/seqid=0 2024-11-16T16:14:27,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c9937e6608ac4317bee6667d962af02b is 50, key is test_row_0/A:col10/1731773666903/Put/seqid=0 2024-11-16T16:14:27,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741886_1062 (size=12663) 2024-11-16T16:14:28,003 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/514e86b528374c9bb9f9a55832e024b1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/514e86b528374c9bb9f9a55832e024b1 2024-11-16T16:14:28,011 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 514e86b528374c9bb9f9a55832e024b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:28,012 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:28,012 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=12, startTime=1731773667834; duration=0sec 2024-11-16T16:14:28,012 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:28,012 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:28,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741887_1063 (size=12151) 2024-11-16T16:14:28,029 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c9937e6608ac4317bee6667d962af02b 2024-11-16T16:14:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b95e88265b24489094e07a030130bb8e is 50, key is test_row_0/B:col10/1731773666903/Put/seqid=0 2024-11-16T16:14:28,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741888_1064 (size=12151) 2024-11-16T16:14:28,062 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b95e88265b24489094e07a030130bb8e 2024-11-16T16:14:28,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4ce94eb06e044d158086db8c89f831dd is 50, key is test_row_0/C:col10/1731773666903/Put/seqid=0 2024-11-16T16:14:28,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741889_1065 (size=12151) 2024-11-16T16:14:28,089 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4ce94eb06e044d158086db8c89f831dd 2024-11-16T16:14:28,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:28,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:28,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c9937e6608ac4317bee6667d962af02b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b 2024-11-16T16:14:28,114 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b, entries=150, sequenceid=227, filesize=11.9 K 2024-11-16T16:14:28,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/b95e88265b24489094e07a030130bb8e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e 2024-11-16T16:14:28,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773728113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773728116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773728118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773728121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,128 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-16T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4ce94eb06e044d158086db8c89f831dd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd 2024-11-16T16:14:28,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773728122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,139 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd, entries=150, sequenceid=227, filesize=11.9 K 2024-11-16T16:14:28,140 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 8161ddf0266b9f8b7a8b291995378f22 in 184ms, sequenceid=227, compaction requested=false 2024-11-16T16:14:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-16T16:14:28,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-16T16:14:28,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-16T16:14:28,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 500 msec 2024-11-16T16:14:28,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 552 msec 2024-11-16T16:14:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:28,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:28,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:28,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/39c6da80b0674798988df3a778e7c7c3 is 50, key is test_row_0/A:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:28,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741890_1066 (size=12151) 2024-11-16T16:14:28,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/39c6da80b0674798988df3a778e7c7c3 2024-11-16T16:14:28,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-16T16:14:28,266 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:28,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-16T16:14:28,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a394e09444a54b109b779492fbce4938 is 50, key is test_row_0/B:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:28,276 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:28,277 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:28,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:28,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741891_1067 (size=12151) 2024-11-16T16:14:28,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773728291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773728292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773728307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773728309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773728309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,321 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e096ab71e5424918adc219dbdd12dda5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e096ab71e5424918adc219dbdd12dda5 2024-11-16T16:14:28,330 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into e096ab71e5424918adc219dbdd12dda5(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:28,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:28,330 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=12, startTime=1731773667834; duration=0sec 2024-11-16T16:14:28,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:28,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:28,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773728415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773728415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773728425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773728425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773728425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:28,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773728625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,628 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773728626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773728635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773728635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,639 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773728635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a394e09444a54b109b779492fbce4938 2024-11-16T16:14:28,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/0398f5971dda45b0b283c63c11ef1a42 is 50, key is test_row_0/C:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:28,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741892_1068 (size=12151) 2024-11-16T16:14:28,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:28,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:28,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773728935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773728935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,939 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773728945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773728945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773728946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:29,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:29,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/0398f5971dda45b0b283c63c11ef1a42 2024-11-16T16:14:29,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/39c6da80b0674798988df3a778e7c7c3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3 2024-11-16T16:14:29,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T16:14:29,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a394e09444a54b109b779492fbce4938 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938 2024-11-16T16:14:29,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T16:14:29,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/0398f5971dda45b0b283c63c11ef1a42 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42 2024-11-16T16:14:29,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42, entries=150, sequenceid=247, filesize=11.9 K 2024-11-16T16:14:29,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8161ddf0266b9f8b7a8b291995378f22 in 909ms, sequenceid=247, compaction requested=true 2024-11-16T16:14:29,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:29,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:29,143 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:29,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:29,143 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:29,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:29,145 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:29,145 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:29,145 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,145 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e096ab71e5424918adc219dbdd12dda5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.1 K 2024-11-16T16:14:29,146 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:29,146 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:29,146 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,146 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e096ab71e5424918adc219dbdd12dda5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666250 2024-11-16T16:14:29,146 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/548fbda844f24e8da8329b22234177b5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.1 K 2024-11-16T16:14:29,146 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b95e88265b24489094e07a030130bb8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731773666903 2024-11-16T16:14:29,146 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 548fbda844f24e8da8329b22234177b5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666250 2024-11-16T16:14:29,147 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9937e6608ac4317bee6667d962af02b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731773666903 2024-11-16T16:14:29,147 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a394e09444a54b109b779492fbce4938, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:29,147 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39c6da80b0674798988df3a778e7c7c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:29,162 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:29,162 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:29,163 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a246cc2e95174c2197ead8525d516424 is 50, key is test_row_0/B:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:29,163 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/3f5a4df792fe4327855d1e24009ba821 is 50, key is test_row_0/A:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:29,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741893_1069 (size=12799) 2024-11-16T16:14:29,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741894_1070 (size=12799) 2024-11-16T16:14:29,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,207 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:29,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:29,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6de4f77165af422fba42207c20747a3b is 50, key is test_row_0/A:col10/1731773668303/Put/seqid=0 2024-11-16T16:14:29,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741895_1071 (size=12301) 2024-11-16T16:14:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:29,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:29,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773729472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773729480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773729483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,488 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773729483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,592 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/3f5a4df792fe4327855d1e24009ba821 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/3f5a4df792fe4327855d1e24009ba821 2024-11-16T16:14:29,592 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/a246cc2e95174c2197ead8525d516424 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a246cc2e95174c2197ead8525d516424 2024-11-16T16:14:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773729595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773729596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773729596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773729596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773729597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,604 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into a246cc2e95174c2197ead8525d516424(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:29,605 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:29,605 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773669143; duration=0sec 2024-11-16T16:14:29,605 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:29,605 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:29,605 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:29,606 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 3f5a4df792fe4327855d1e24009ba821(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:29,606 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:29,606 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773669143; duration=0sec 2024-11-16T16:14:29,606 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:29,606 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:29,607 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:29,607 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:29,607 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:29,607 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/514e86b528374c9bb9f9a55832e024b1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.1 K 2024-11-16T16:14:29,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 514e86b528374c9bb9f9a55832e024b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731773666250 2024-11-16T16:14:29,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ce94eb06e044d158086db8c89f831dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731773666903 2024-11-16T16:14:29,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0398f5971dda45b0b283c63c11ef1a42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:29,619 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#59 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:29,619 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6de4f77165af422fba42207c20747a3b 2024-11-16T16:14:29,620 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e1a8a015968a4bd888ad5eba13afa92f is 50, key is test_row_0/C:col10/1731773668230/Put/seqid=0 2024-11-16T16:14:29,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741896_1072 (size=12765) 2024-11-16T16:14:29,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/0c074750bde944858e5c7ced5d6f4fbc is 50, key is test_row_0/B:col10/1731773668303/Put/seqid=0 2024-11-16T16:14:29,641 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e1a8a015968a4bd888ad5eba13afa92f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e1a8a015968a4bd888ad5eba13afa92f 2024-11-16T16:14:29,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741897_1073 (size=12301) 2024-11-16T16:14:29,651 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/0c074750bde944858e5c7ced5d6f4fbc 2024-11-16T16:14:29,652 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into e1a8a015968a4bd888ad5eba13afa92f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:29,652 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:29,652 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773669144; duration=0sec 2024-11-16T16:14:29,652 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:29,652 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:29,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c52f7f158cef46c7a928a207dc5123d2 is 50, key is test_row_0/C:col10/1731773668303/Put/seqid=0 2024-11-16T16:14:29,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741898_1074 (size=12301) 2024-11-16T16:14:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773729805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773729805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773729805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773729805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773729805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,068 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c52f7f158cef46c7a928a207dc5123d2 2024-11-16T16:14:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/6de4f77165af422fba42207c20747a3b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b 2024-11-16T16:14:30,081 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T16:14:30,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/0c074750bde944858e5c7ced5d6f4fbc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc 2024-11-16T16:14:30,089 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T16:14:30,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c52f7f158cef46c7a928a207dc5123d2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2 2024-11-16T16:14:30,099 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2, entries=150, sequenceid=262, filesize=12.0 K 2024-11-16T16:14:30,101 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 8161ddf0266b9f8b7a8b291995378f22 in 893ms, sequenceid=262, compaction requested=false 2024-11-16T16:14:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-16T16:14:30,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-16T16:14:30,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-16T16:14:30,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8250 sec 2024-11-16T16:14:30,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8350 sec 2024-11-16T16:14:30,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:30,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-16T16:14:30,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:30,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/2e4f133cfe2e48d680d9eca07c43386d is 50, key is test_row_0/A:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773730134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773730135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741899_1075 (size=12301) 2024-11-16T16:14:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773730145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/2e4f133cfe2e48d680d9eca07c43386d 2024-11-16T16:14:30,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773730147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773730148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/2b2a7303425a40999beee29decab7d11 is 50, key is test_row_0/B:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741900_1076 (size=12301) 2024-11-16T16:14:30,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/2b2a7303425a40999beee29decab7d11 2024-11-16T16:14:30,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/ea211da2aa6441c587c8665e196103a1 is 50, key is test_row_0/C:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741901_1077 (size=12301) 2024-11-16T16:14:30,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/ea211da2aa6441c587c8665e196103a1 2024-11-16T16:14:30,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/2e4f133cfe2e48d680d9eca07c43386d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d 2024-11-16T16:14:30,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-16T16:14:30,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/2b2a7303425a40999beee29decab7d11 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11 2024-11-16T16:14:30,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773730255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773730255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11, entries=150, sequenceid=286, filesize=12.0 K 2024-11-16T16:14:30,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773730256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,263 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773730256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773730256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/ea211da2aa6441c587c8665e196103a1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1 2024-11-16T16:14:30,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1, entries=150, sequenceid=286, filesize=12.0 K 2024-11-16T16:14:30,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8161ddf0266b9f8b7a8b291995378f22 in 153ms, sequenceid=286, compaction requested=true 2024-11-16T16:14:30,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:30,275 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:30,275 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:30,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:30,276 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:30,276 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:30,276 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,277 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/3f5a4df792fe4327855d1e24009ba821, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.5 K 2024-11-16T16:14:30,277 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:30,277 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:30,277 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,278 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a246cc2e95174c2197ead8525d516424, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.5 K 2024-11-16T16:14:30,278 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f5a4df792fe4327855d1e24009ba821, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:30,279 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6de4f77165af422fba42207c20747a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731773668302 2024-11-16T16:14:30,279 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a246cc2e95174c2197ead8525d516424, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:30,279 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e4f133cfe2e48d680d9eca07c43386d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:30,279 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c074750bde944858e5c7ced5d6f4fbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731773668302 2024-11-16T16:14:30,280 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b2a7303425a40999beee29decab7d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:30,299 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:30,300 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/47eed913216e42f6856c00ee67a7e1a5 is 50, key is test_row_0/A:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,311 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:30,311 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a is 50, key is test_row_0/B:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741902_1078 (size=13051) 2024-11-16T16:14:30,344 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/47eed913216e42f6856c00ee67a7e1a5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/47eed913216e42f6856c00ee67a7e1a5 2024-11-16T16:14:30,352 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 47eed913216e42f6856c00ee67a7e1a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:30,352 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:30,352 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773670274; duration=0sec 2024-11-16T16:14:30,352 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:30,352 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:30,353 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:30,354 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:30,354 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:30,354 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,354 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e1a8a015968a4bd888ad5eba13afa92f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.5 K 2024-11-16T16:14:30,355 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1a8a015968a4bd888ad5eba13afa92f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731773668106 2024-11-16T16:14:30,356 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c52f7f158cef46c7a928a207dc5123d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731773668302 2024-11-16T16:14:30,357 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea211da2aa6441c587c8665e196103a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:30,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741903_1079 (size=13051) 2024-11-16T16:14:30,370 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:30,371 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/a2eef14e18cb4d22b8c053d53926fee0 is 50, key is test_row_0/C:col10/1731773669479/Put/seqid=0 2024-11-16T16:14:30,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741904_1080 (size=13017) 2024-11-16T16:14:30,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-16T16:14:30,425 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:30,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-16T16:14:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T16:14:30,427 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:30,428 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:30,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:30,467 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:30,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:30,472 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/81e8743a52e54d8997182b88efd656fe is 50, key is test_row_0/A:col10/1731773670142/Put/seqid=0 2024-11-16T16:14:30,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741905_1081 (size=14741) 2024-11-16T16:14:30,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/81e8743a52e54d8997182b88efd656fe 2024-11-16T16:14:30,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773730497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773730498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773730500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,511 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773730500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773730508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ffbfdf434a8443cf86cc4797348689f5 is 50, key is test_row_0/B:col10/1731773670142/Put/seqid=0 2024-11-16T16:14:30,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741906_1082 (size=12301) 2024-11-16T16:14:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T16:14:30,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T16:14:30,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:30,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773730615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773730615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773730616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773730616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773730625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T16:14:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T16:14:30,773 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a 2024-11-16T16:14:30,779 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 8a4d0d8b7b3143fd8f2ff0d23643bd6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:30,779 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:30,779 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773670275; duration=0sec 2024-11-16T16:14:30,779 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:30,779 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:30,785 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/a2eef14e18cb4d22b8c053d53926fee0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/a2eef14e18cb4d22b8c053d53926fee0 2024-11-16T16:14:30,792 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into a2eef14e18cb4d22b8c053d53926fee0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:30,792 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:30,792 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773670275; duration=0sec 2024-11-16T16:14:30,792 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:30,792 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773730824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773730824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773730835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773730835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773730835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:30,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:30,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T16:14:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:30,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:30,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ffbfdf434a8443cf86cc4797348689f5 2024-11-16T16:14:30,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c07e9293e481478eb77411a010a669f6 is 50, key is test_row_0/C:col10/1731773670142/Put/seqid=0 2024-11-16T16:14:30,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741907_1083 (size=12301) 2024-11-16T16:14:30,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c07e9293e481478eb77411a010a669f6 2024-11-16T16:14:30,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/81e8743a52e54d8997182b88efd656fe as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe 2024-11-16T16:14:30,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe, entries=200, sequenceid=299, filesize=14.4 K 2024-11-16T16:14:30,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/ffbfdf434a8443cf86cc4797348689f5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5 2024-11-16T16:14:30,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5, entries=150, sequenceid=299, filesize=12.0 K 2024-11-16T16:14:30,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/c07e9293e481478eb77411a010a669f6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6 2024-11-16T16:14:30,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6, entries=150, sequenceid=299, filesize=12.0 K 2024-11-16T16:14:30,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8161ddf0266b9f8b7a8b291995378f22 in 512ms, sequenceid=299, compaction requested=false 2024-11-16T16:14:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:31,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:31,046 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:31,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d18027d89ae74b69bc5c48d8d1a29446 is 50, key is test_row_0/A:col10/1731773670498/Put/seqid=0 2024-11-16T16:14:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T16:14:31,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741908_1084 (size=12301) 2024-11-16T16:14:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:31,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:31,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773731147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773731149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773731151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773731152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,157 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773731153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773731254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,258 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773731265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773731265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773731265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773731265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,463 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d18027d89ae74b69bc5c48d8d1a29446 2024-11-16T16:14:31,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/609c32eb88e54fe4a2238b632ce5ca07 is 50, key is test_row_0/B:col10/1731773670498/Put/seqid=0 2024-11-16T16:14:31,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773731465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741909_1085 (size=12301) 2024-11-16T16:14:31,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773731475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773731476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773731476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773731476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,478 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/609c32eb88e54fe4a2238b632ce5ca07 2024-11-16T16:14:31,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6900bde1545d4556bc2c609b0a70416b is 50, key is test_row_0/C:col10/1731773670498/Put/seqid=0 2024-11-16T16:14:31,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741910_1086 (size=12301) 2024-11-16T16:14:31,501 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6900bde1545d4556bc2c609b0a70416b 2024-11-16T16:14:31,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d18027d89ae74b69bc5c48d8d1a29446 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446 2024-11-16T16:14:31,519 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T16:14:31,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/609c32eb88e54fe4a2238b632ce5ca07 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07 2024-11-16T16:14:31,527 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T16:14:31,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/6900bde1545d4556bc2c609b0a70416b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b 2024-11-16T16:14:31,538 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b, entries=150, sequenceid=321, filesize=12.0 K 2024-11-16T16:14:31,539 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 8161ddf0266b9f8b7a8b291995378f22 in 493ms, sequenceid=321, compaction requested=true 2024-11-16T16:14:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:31,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-16T16:14:31,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-16T16:14:31,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-16T16:14:31,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-16T16:14:31,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1200 sec 2024-11-16T16:14:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-16T16:14:31,565 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:31,566 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-16T16:14:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T16:14:31,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:31,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:31,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T16:14:31,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-16T16:14:31,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:31,722 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ee0bb4d5b07b4096a65c3a77f8c0533c is 50, key is test_row_0/A:col10/1731773671150/Put/seqid=0 2024-11-16T16:14:31,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741911_1087 (size=12301) 2024-11-16T16:14:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:31,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:31,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773731800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773731802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773731804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773731805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773731806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T16:14:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773731915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773731916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773731916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773731916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773731916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:31,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773732126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773732126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773732126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773732126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773732126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,138 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ee0bb4d5b07b4096a65c3a77f8c0533c 2024-11-16T16:14:32,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/81279ad07b3e4abaabf57a4f4a8b525a is 50, key is test_row_0/B:col10/1731773671150/Put/seqid=0 2024-11-16T16:14:32,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741912_1088 (size=12301) 2024-11-16T16:14:32,165 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/81279ad07b3e4abaabf57a4f4a8b525a 2024-11-16T16:14:32,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/b46dab7a7d8144ceb0c5b04eedf314b3 is 50, key is test_row_0/C:col10/1731773671150/Put/seqid=0 2024-11-16T16:14:32,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741913_1089 (size=12301) 2024-11-16T16:14:32,194 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/b46dab7a7d8144ceb0c5b04eedf314b3 2024-11-16T16:14:32,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T16:14:32,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ee0bb4d5b07b4096a65c3a77f8c0533c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c 2024-11-16T16:14:32,208 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c, entries=150, sequenceid=334, filesize=12.0 K 2024-11-16T16:14:32,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/81279ad07b3e4abaabf57a4f4a8b525a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a 2024-11-16T16:14:32,216 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a, entries=150, sequenceid=334, filesize=12.0 K 2024-11-16T16:14:32,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/b46dab7a7d8144ceb0c5b04eedf314b3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3 2024-11-16T16:14:32,226 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3, entries=150, sequenceid=334, filesize=12.0 K 2024-11-16T16:14:32,227 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8161ddf0266b9f8b7a8b291995378f22 in 505ms, sequenceid=334, compaction requested=true 2024-11-16T16:14:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-16T16:14:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-16T16:14:32,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-16T16:14:32,231 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 660 msec 2024-11-16T16:14:32,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 666 msec 2024-11-16T16:14:32,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:32,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:32,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:32,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/5decede801914864bfa514fc9adf96e1 is 50, key is test_row_0/A:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:32,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773732460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741914_1090 (size=17181) 2024-11-16T16:14:32,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/5decede801914864bfa514fc9adf96e1 2024-11-16T16:14:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773732461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773732463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773732464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773732464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/89223bc9eb02405c9ee64e6132b7b862 is 50, key is test_row_0/B:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:32,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741915_1091 (size=12301) 2024-11-16T16:14:32,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773732575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773732585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773732585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773732585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773732585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-16T16:14:32,705 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:32,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:32,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-16T16:14:32,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T16:14:32,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:32,730 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:32,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:32,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773732795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773732795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773732795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773732795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773732795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:32,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T16:14:32,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T16:14:32,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:32,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:32,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:32,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:32,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/89223bc9eb02405c9ee64e6132b7b862 2024-11-16T16:14:32,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/58a216c7dd8a4b24841d2d7111c2ab08 is 50, key is test_row_0/C:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741916_1092 (size=12301) 2024-11-16T16:14:32,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/58a216c7dd8a4b24841d2d7111c2ab08 2024-11-16T16:14:32,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/5decede801914864bfa514fc9adf96e1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1 2024-11-16T16:14:32,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1, entries=250, sequenceid=355, filesize=16.8 K 2024-11-16T16:14:32,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/89223bc9eb02405c9ee64e6132b7b862 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862 2024-11-16T16:14:32,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862, entries=150, sequenceid=355, filesize=12.0 K 2024-11-16T16:14:32,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/58a216c7dd8a4b24841d2d7111c2ab08 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08 2024-11-16T16:14:32,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08, entries=150, sequenceid=355, filesize=12.0 K 2024-11-16T16:14:32,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 8161ddf0266b9f8b7a8b291995378f22 in 496ms, sequenceid=355, compaction requested=true 2024-11-16T16:14:32,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:32,946 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:32,946 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:32,949 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 69575 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T16:14:32,949 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62255 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T16:14:32,949 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:32,949 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:32,949 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,949 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,949 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=60.8 K 2024-11-16T16:14:32,949 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/47eed913216e42f6856c00ee67a7e1a5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=67.9 K 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a4d0d8b7b3143fd8f2ff0d23643bd6a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47eed913216e42f6856c00ee67a7e1a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbfdf434a8443cf86cc4797348689f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731773670130 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81e8743a52e54d8997182b88efd656fe, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731773670130 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 609c32eb88e54fe4a2238b632ce5ca07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731773670497 2024-11-16T16:14:32,950 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d18027d89ae74b69bc5c48d8d1a29446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731773670497 2024-11-16T16:14:32,951 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 81279ad07b3e4abaabf57a4f4a8b525a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731773671149 2024-11-16T16:14:32,951 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 89223bc9eb02405c9ee64e6132b7b862, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:32,952 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee0bb4d5b07b4096a65c3a77f8c0533c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731773671149 2024-11-16T16:14:32,953 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5decede801914864bfa514fc9adf96e1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:32,970 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#80 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:32,970 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#81 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:32,970 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/bf95d6177ccd491792e80a2da4b53379 is 50, key is test_row_0/A:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:32,971 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/41aa3846591f408f97bf06f13da49230 is 50, key is test_row_0/B:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:32,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741917_1093 (size=13221) 2024-11-16T16:14:32,981 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/41aa3846591f408f97bf06f13da49230 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41aa3846591f408f97bf06f13da49230 2024-11-16T16:14:32,991 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 41aa3846591f408f97bf06f13da49230(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:32,991 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:32,991 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=11, startTime=1731773672946; duration=0sec 2024-11-16T16:14:32,991 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:32,991 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:32,991 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-16T16:14:32,993 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62221 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-16T16:14:32,993 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:32,993 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:32,993 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/a2eef14e18cb4d22b8c053d53926fee0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=60.8 K 2024-11-16T16:14:32,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a2eef14e18cb4d22b8c053d53926fee0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731773669479 2024-11-16T16:14:32,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c07e9293e481478eb77411a010a669f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731773670130 2024-11-16T16:14:32,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6900bde1545d4556bc2c609b0a70416b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=321, earliestPutTs=1731773670497 2024-11-16T16:14:32,995 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b46dab7a7d8144ceb0c5b04eedf314b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731773671149 2024-11-16T16:14:32,995 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 58a216c7dd8a4b24841d2d7111c2ab08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:32,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741918_1094 (size=13221) 2024-11-16T16:14:33,010 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/bf95d6177ccd491792e80a2da4b53379 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bf95d6177ccd491792e80a2da4b53379 2024-11-16T16:14:33,018 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into bf95d6177ccd491792e80a2da4b53379(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:33,018 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:33,018 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=11, startTime=1731773672946; duration=0sec 2024-11-16T16:14:33,018 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:33,018 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:33,025 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#82 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:33,025 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/eff1c7529f5d446987dc9fa26610afb6 is 50, key is test_row_0/C:col10/1731773672449/Put/seqid=0 2024-11-16T16:14:33,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-16T16:14:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:33,036 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:33,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741919_1095 (size=13187) 2024-11-16T16:14:33,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T16:14:33,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/41a70067ce394656a78f42153921701b is 50, key is test_row_0/A:col10/1731773672463/Put/seqid=0 2024-11-16T16:14:33,047 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/eff1c7529f5d446987dc9fa26610afb6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/eff1c7529f5d446987dc9fa26610afb6 2024-11-16T16:14:33,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741920_1096 (size=12301) 2024-11-16T16:14:33,055 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/41a70067ce394656a78f42153921701b 2024-11-16T16:14:33,056 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into eff1c7529f5d446987dc9fa26610afb6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:33,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:33,056 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=11, startTime=1731773672946; duration=0sec 2024-11-16T16:14:33,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:33,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:33,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e0909cb71cbf44b1959d920d06382592 is 50, key is test_row_0/B:col10/1731773672463/Put/seqid=0 2024-11-16T16:14:33,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741921_1097 (size=12301) 2024-11-16T16:14:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:33,136 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773673123, max timestamp=1731773673130Num uniques 859; 2024-11-16T16:14:33,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773673123, max timestamp=1731773673130Num uniques 859; 2024-11-16T16:14:33,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773673123, max timestamp=1731773673130Num uniques 859; 2024-11-16T16:14:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:33,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:33,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773733157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773733158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773733160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773733161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773733172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773733285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773733285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773733285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773733285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773733285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T16:14:33,479 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e0909cb71cbf44b1959d920d06382592 2024-11-16T16:14:33,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/94586f4b37404892905f9dad8310ee10 is 50, key is test_row_0/C:col10/1731773672463/Put/seqid=0 2024-11-16T16:14:33,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773733495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773733495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773733495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773733505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773733505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741922_1098 (size=12301) 2024-11-16T16:14:33,516 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/94586f4b37404892905f9dad8310ee10 2024-11-16T16:14:33,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/41a70067ce394656a78f42153921701b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b 2024-11-16T16:14:33,527 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T16:14:33,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e0909cb71cbf44b1959d920d06382592 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592 2024-11-16T16:14:33,533 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T16:14:33,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/94586f4b37404892905f9dad8310ee10 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10 2024-11-16T16:14:33,540 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T16:14:33,541 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8161ddf0266b9f8b7a8b291995378f22 in 505ms, sequenceid=368, compaction requested=false 2024-11-16T16:14:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-16T16:14:33,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-16T16:14:33,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-16T16:14:33,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 812 msec 2024-11-16T16:14:33,548 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 841 msec 2024-11-16T16:14:33,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:33,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:33,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:14:33,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/63ed71c4031748579dc66af595b8c048 is 50, key is test_row_0/A:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741923_1099 (size=12301) 2024-11-16T16:14:33,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/63ed71c4031748579dc66af595b8c048 2024-11-16T16:14:33,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773733824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773733826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,838 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773733835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773733837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773733838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e5c9c532740f4d94810b060d8071e7a7 is 50, key is test_row_0/B:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741924_1100 (size=12301) 2024-11-16T16:14:33,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e5c9c532740f4d94810b060d8071e7a7 2024-11-16T16:14:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-16T16:14:33,865 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:33,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-16T16:14:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:33,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/48046148f6e34be5899ca9437750978b is 50, key is test_row_0/C:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:33,871 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:33,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:33,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741925_1101 (size=12301) 2024-11-16T16:14:33,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/48046148f6e34be5899ca9437750978b 2024-11-16T16:14:33,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/63ed71c4031748579dc66af595b8c048 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048 2024-11-16T16:14:33,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T16:14:33,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/e5c9c532740f4d94810b060d8071e7a7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7 2024-11-16T16:14:33,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T16:14:33,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/48046148f6e34be5899ca9437750978b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b 2024-11-16T16:14:33,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b, entries=150, sequenceid=407, filesize=12.0 K 2024-11-16T16:14:33,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=46.96 KB/48090 for 8161ddf0266b9f8b7a8b291995378f22 in 94ms, sequenceid=407, compaction requested=true 2024-11-16T16:14:33,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:33,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:33,914 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:33,914 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:33,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:33,915 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:33,915 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:33,915 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:33,915 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:33,915 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:33,915 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:33,915 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41aa3846591f408f97bf06f13da49230, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.9 K 2024-11-16T16:14:33,915 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bf95d6177ccd491792e80a2da4b53379, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.9 K 2024-11-16T16:14:33,917 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf95d6177ccd491792e80a2da4b53379, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:33,917 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 41aa3846591f408f97bf06f13da49230, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:33,917 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41a70067ce394656a78f42153921701b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773672455 2024-11-16T16:14:33,917 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e0909cb71cbf44b1959d920d06382592, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773672455 2024-11-16T16:14:33,918 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ed71c4031748579dc66af595b8c048, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:33,918 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e5c9c532740f4d94810b060d8071e7a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:33,930 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#89 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:33,930 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:33,930 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9082add2968d43499b77fcc7a19b6762 is 50, key is test_row_0/A:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,930 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/c509f258cdef4a1db39c12dd999c6318 is 50, key is test_row_0/B:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741926_1102 (size=13323) 2024-11-16T16:14:33,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741927_1103 (size=13323) 2024-11-16T16:14:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:33,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:33,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:33,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/288422b6d60f4a3e99cff5e288691bac is 50, key is test_row_0/A:col10/1731773673946/Put/seqid=0 2024-11-16T16:14:33,959 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9082add2968d43499b77fcc7a19b6762 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9082add2968d43499b77fcc7a19b6762 2024-11-16T16:14:33,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741928_1104 (size=17181) 2024-11-16T16:14:33,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/288422b6d60f4a3e99cff5e288691bac 2024-11-16T16:14:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:33,976 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 9082add2968d43499b77fcc7a19b6762(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:33,977 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:33,977 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773673913; duration=0sec 2024-11-16T16:14:33,977 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:33,977 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:33,977 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:33,980 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:33,980 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:33,980 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:33,981 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/eff1c7529f5d446987dc9fa26610afb6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=36.9 K 2024-11-16T16:14:33,981 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting eff1c7529f5d446987dc9fa26610afb6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1731773671802 2024-11-16T16:14:33,982 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94586f4b37404892905f9dad8310ee10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773672455 2024-11-16T16:14:33,982 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48046148f6e34be5899ca9437750978b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:33,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8802cd4225784d0f99fb9a43ec296153 is 50, key is test_row_0/B:col10/1731773673946/Put/seqid=0 2024-11-16T16:14:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773733976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773733976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773733978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:33,993 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:33,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773673969, max timestamp=1731773673987Num uniques 859; 2024-11-16T16:14:33,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:14:33,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773673969, max timestamp=1731773673987Num uniques 859; 2024-11-16T16:14:33,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:33,994 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/0ef1b73106e644b4ada81eb68bf2ac32 is 50, key is test_row_0/C:col10/1731773673158/Put/seqid=0 2024-11-16T16:14:33,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773673969, max timestamp=1731773673987Num uniques 859; 2024-11-16T16:14:33,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:14:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:33,997 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741929_1105 (size=12301) 2024-11-16T16:14:34,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8802cd4225784d0f99fb9a43ec296153 2024-11-16T16:14:34,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741930_1106 (size=13289) 2024-11-16T16:14:34,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/889d63f511dd45a29ad3a72fae75c9c1 is 50, key is test_row_0/C:col10/1731773673946/Put/seqid=0 2024-11-16T16:14:34,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741931_1107 (size=12301) 2024-11-16T16:14:34,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T16:14:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773734090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773734090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773734105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773734105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773734105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:34,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T16:14:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:34,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773734195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773734195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773734315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773734316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773734316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T16:14:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:34,357 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/c509f258cdef4a1db39c12dd999c6318 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/c509f258cdef4a1db39c12dd999c6318 2024-11-16T16:14:34,365 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into c509f258cdef4a1db39c12dd999c6318(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:34,365 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:34,365 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773673914; duration=0sec 2024-11-16T16:14:34,365 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:34,365 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:34,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773734415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773734415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,421 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/0ef1b73106e644b4ada81eb68bf2ac32 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0ef1b73106e644b4ada81eb68bf2ac32 2024-11-16T16:14:34,427 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 0ef1b73106e644b4ada81eb68bf2ac32(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:34,427 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:34,427 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773673914; duration=0sec 2024-11-16T16:14:34,427 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:34,427 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:34,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/889d63f511dd45a29ad3a72fae75c9c1 2024-11-16T16:14:34,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/288422b6d60f4a3e99cff5e288691bac as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac 2024-11-16T16:14:34,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac, entries=250, sequenceid=419, filesize=16.8 K 2024-11-16T16:14:34,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8802cd4225784d0f99fb9a43ec296153 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153 2024-11-16T16:14:34,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153, entries=150, sequenceid=419, filesize=12.0 K 2024-11-16T16:14:34,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/889d63f511dd45a29ad3a72fae75c9c1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1 2024-11-16T16:14:34,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1, entries=150, sequenceid=419, filesize=12.0 K 2024-11-16T16:14:34,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=123.00 KB/125950 for 8161ddf0266b9f8b7a8b291995378f22 in 515ms, sequenceid=419, compaction requested=false 2024-11-16T16:14:34,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:34,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-16T16:14:34,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:34,493 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=123.00 KB heapSize=345.53 KB 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:34,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:34,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:34,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9ffff6b0e0ff492686f9e461ccff322d is 50, key is test_row_0/A:col10/1731773674074/Put/seqid=0 2024-11-16T16:14:34,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741932_1108 (size=12301) 2024-11-16T16:14:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:34,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773734649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773734651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773734652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773734725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773734726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773734755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773734765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773734765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,905 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9ffff6b0e0ff492686f9e461ccff322d 2024-11-16T16:14:34,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/5d6a3310b81d4d768d337af8ffc43443 is 50, key is test_row_0/B:col10/1731773674074/Put/seqid=0 2024-11-16T16:14:34,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741933_1109 (size=12301) 2024-11-16T16:14:34,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773734965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773734975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773734975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:34,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773735244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773735244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773735275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773735285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773735285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,317 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/5d6a3310b81d4d768d337af8ffc43443 2024-11-16T16:14:35,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45e450f6df93425ea228771af459ff99 is 50, key is test_row_0/C:col10/1731773674074/Put/seqid=0 2024-11-16T16:14:35,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741934_1110 (size=12301) 2024-11-16T16:14:35,733 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45e450f6df93425ea228771af459ff99 2024-11-16T16:14:35,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9ffff6b0e0ff492686f9e461ccff322d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d 2024-11-16T16:14:35,743 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T16:14:35,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/5d6a3310b81d4d768d337af8ffc43443 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443 2024-11-16T16:14:35,750 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T16:14:35,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45e450f6df93425ea228771af459ff99 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99 2024-11-16T16:14:35,756 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T16:14:35,757 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~123.00 KB/125950, heapSize ~345.48 KB/353776, currentSize=60.38 KB/61830 for 8161ddf0266b9f8b7a8b291995378f22 in 1264ms, sequenceid=456, compaction requested=true 2024-11-16T16:14:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-16T16:14:35,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-16T16:14:35,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-16T16:14:35,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8870 sec 2024-11-16T16:14:35,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.8960 sec 2024-11-16T16:14:35,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:35,788 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:35,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:35,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ccb0ef8005894c62a12b76c5791bfc0f is 50, key is test_row_0/A:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:35,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741935_1111 (size=14741) 2024-11-16T16:14:35,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ccb0ef8005894c62a12b76c5791bfc0f 2024-11-16T16:14:35,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4cec96196c8c411fb140370597e5dd53 is 50, key is test_row_0/B:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:35,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741936_1112 (size=12301) 2024-11-16T16:14:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773735820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773735824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773735825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,834 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773735935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773735935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:35,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:35,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773735945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:35,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-16T16:14:36,015 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:36,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:36,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-16T16:14:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:36,018 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:36,019 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:36,019 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:36,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773736145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773736155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773736155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T16:14:36,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:36,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4cec96196c8c411fb140370597e5dd53 2024-11-16T16:14:36,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7c6ac7126c6c4270ad6c0f67cc7a4588 is 50, key is test_row_0/C:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:36,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741937_1113 (size=12301) 2024-11-16T16:14:36,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773736255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,261 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 8161ddf0266b9f8b7a8b291995378f22 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773736265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,266 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 8161ddf0266b9f8b7a8b291995378f22 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:36,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T16:14:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:36,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773736455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773736465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773736465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T16:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T16:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:36,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7c6ac7126c6c4270ad6c0f67cc7a4588 2024-11-16T16:14:36,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/ccb0ef8005894c62a12b76c5791bfc0f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f 2024-11-16T16:14:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:36,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f, entries=200, sequenceid=469, filesize=14.4 K 2024-11-16T16:14:36,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4cec96196c8c411fb140370597e5dd53 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53 2024-11-16T16:14:36,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T16:14:36,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7c6ac7126c6c4270ad6c0f67cc7a4588 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588 2024-11-16T16:14:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588, entries=150, sequenceid=469, filesize=12.0 K 2024-11-16T16:14:36,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8161ddf0266b9f8b7a8b291995378f22 in 880ms, sequenceid=469, compaction requested=true 2024-11-16T16:14:36,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:36,668 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:36,669 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:36,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:36,670 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:36,670 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:36,670 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:36,670 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:36,670 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,670 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,671 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/c509f258cdef4a1db39c12dd999c6318, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=49.0 K 2024-11-16T16:14:36,671 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9082add2968d43499b77fcc7a19b6762, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=56.2 K 2024-11-16T16:14:36,671 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9082add2968d43499b77fcc7a19b6762, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:36,671 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c509f258cdef4a1db39c12dd999c6318, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:36,672 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288422b6d60f4a3e99cff5e288691bac, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731773673821 2024-11-16T16:14:36,672 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8802cd4225784d0f99fb9a43ec296153, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731773673823 2024-11-16T16:14:36,672 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ffff6b0e0ff492686f9e461ccff322d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773674074 2024-11-16T16:14:36,672 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d6a3310b81d4d768d337af8ffc43443, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773674074 2024-11-16T16:14:36,672 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccb0ef8005894c62a12b76c5791bfc0f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674644 2024-11-16T16:14:36,673 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cec96196c8c411fb140370597e5dd53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674647 2024-11-16T16:14:36,695 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:36,696 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/5d91b66b70064cdbb4e766e58b380fcb is 50, key is test_row_0/A:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:36,706 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:36,707 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/41ad33ebd4c047e88cb73e351e2bf59c is 50, key is test_row_0/B:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:36,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741938_1114 (size=13459) 2024-11-16T16:14:36,717 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/5d91b66b70064cdbb4e766e58b380fcb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5d91b66b70064cdbb4e766e58b380fcb 2024-11-16T16:14:36,725 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 5d91b66b70064cdbb4e766e58b380fcb(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:36,725 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:36,725 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=12, startTime=1731773676668; duration=0sec 2024-11-16T16:14:36,725 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:36,725 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:36,725 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:36,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741939_1115 (size=13459) 2024-11-16T16:14:36,729 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:36,729 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:36,729 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,729 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0ef1b73106e644b4ada81eb68bf2ac32, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=49.0 K 2024-11-16T16:14:36,730 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ef1b73106e644b4ada81eb68bf2ac32, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1731773673158 2024-11-16T16:14:36,730 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 889d63f511dd45a29ad3a72fae75c9c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731773673823 2024-11-16T16:14:36,731 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45e450f6df93425ea228771af459ff99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773674074 2024-11-16T16:14:36,734 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c6ac7126c6c4270ad6c0f67cc7a4588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674647 2024-11-16T16:14:36,738 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/41ad33ebd4c047e88cb73e351e2bf59c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41ad33ebd4c047e88cb73e351e2bf59c 2024-11-16T16:14:36,747 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 41ad33ebd4c047e88cb73e351e2bf59c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:36,747 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:36,747 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=12, startTime=1731773676669; duration=0sec 2024-11-16T16:14:36,747 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:36,747 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:36,759 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#103 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:36,760 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4a62dfbe60e442aea2d3995e99ae6bf8 is 50, key is test_row_0/C:col10/1731773675786/Put/seqid=0 2024-11-16T16:14:36,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741940_1116 (size=13425) 2024-11-16T16:14:36,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-16T16:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:36,787 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:36,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:36,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/27e2d236e92c4c4ca1f76f04f1b1e0cd is 50, key is test_row_0/A:col10/1731773675822/Put/seqid=0 2024-11-16T16:14:36,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741941_1117 (size=12301) 2024-11-16T16:14:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:36,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:36,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773736986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773736993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,994 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:36,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:36,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773736994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:36,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:36,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773737094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773737104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773737105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:37,179 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/4a62dfbe60e442aea2d3995e99ae6bf8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4a62dfbe60e442aea2d3995e99ae6bf8 2024-11-16T16:14:37,187 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 4a62dfbe60e442aea2d3995e99ae6bf8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:37,187 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:37,187 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=12, startTime=1731773676669; duration=0sec 2024-11-16T16:14:37,187 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:37,187 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:37,226 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/27e2d236e92c4c4ca1f76f04f1b1e0cd 2024-11-16T16:14:37,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f76e0600638342f989571c6feeeecfc9 is 50, key is test_row_0/B:col10/1731773675822/Put/seqid=0 2024-11-16T16:14:37,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741942_1118 (size=12301) 2024-11-16T16:14:37,245 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f76e0600638342f989571c6feeeecfc9 2024-11-16T16:14:37,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/f3d61a157f81488fae45a4f38a6b9f15 is 50, key is test_row_0/C:col10/1731773675822/Put/seqid=0 2024-11-16T16:14:37,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741943_1119 (size=12301) 2024-11-16T16:14:37,273 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/f3d61a157f81488fae45a4f38a6b9f15 2024-11-16T16:14:37,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/27e2d236e92c4c4ca1f76f04f1b1e0cd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd 2024-11-16T16:14:37,287 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T16:14:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f76e0600638342f989571c6feeeecfc9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9 2024-11-16T16:14:37,303 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T16:14:37,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/f3d61a157f81488fae45a4f38a6b9f15 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15 2024-11-16T16:14:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773737305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,312 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15, entries=150, sequenceid=490, filesize=12.0 K 2024-11-16T16:14:37,313 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 8161ddf0266b9f8b7a8b291995378f22 in 526ms, sequenceid=490, compaction requested=false 2024-11-16T16:14:37,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:37,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:37,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-16T16:14:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-16T16:14:37,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-16T16:14:37,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2970 sec 2024-11-16T16:14:37,322 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.3040 sec 2024-11-16T16:14:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:37,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:37,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:37,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/03ebb4e35acf4bc68bf3db100bb39684 is 50, key is test_row_0/A:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:37,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741944_1120 (size=17181) 2024-11-16T16:14:37,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/03ebb4e35acf4bc68bf3db100bb39684 2024-11-16T16:14:37,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/51fbfd401b314e56a973f247bca30dc1 is 50, key is test_row_0/B:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:37,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741945_1121 (size=12301) 2024-11-16T16:14:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773737383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773737384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773737495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773737495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773737614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773737705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:37,708 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773737705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:37,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:37,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/51fbfd401b314e56a973f247bca30dc1 2024-11-16T16:14:37,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7152475810fc494bacd3a34b75362019 is 50, key is test_row_0/C:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:37,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741946_1122 (size=12301) 2024-11-16T16:14:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773738015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773738015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773738124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-16T16:14:38,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:38,166 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees 2024-11-16T16:14:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T16:14:38,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:38,169 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=29, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:38,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:38,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7152475810fc494bacd3a34b75362019 2024-11-16T16:14:38,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T16:14:38,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60688 deadline: 1731773738275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/03ebb4e35acf4bc68bf3db100bb39684 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684 2024-11-16T16:14:38,278 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 8161ddf0266b9f8b7a8b291995378f22 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684, entries=250, sequenceid=507, filesize=16.8 K 2024-11-16T16:14:38,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/51fbfd401b314e56a973f247bca30dc1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1 2024-11-16T16:14:38,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1, entries=150, sequenceid=507, filesize=12.0 K 2024-11-16T16:14:38,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/7152475810fc494bacd3a34b75362019 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019 2024-11-16T16:14:38,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60718 deadline: 1731773738295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,297 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 8161ddf0266b9f8b7a8b291995378f22 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019, entries=150, sequenceid=507, filesize=12.0 K 2024-11-16T16:14:38,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8161ddf0266b9f8b7a8b291995378f22 in 962ms, sequenceid=507, compaction requested=true 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:38,300 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:38,300 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:38,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:38,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:38,302 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:38,302 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:38,302 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:38,302 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5d91b66b70064cdbb4e766e58b380fcb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=41.9 K 2024-11-16T16:14:38,302 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:38,303 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:38,303 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:38,303 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41ad33ebd4c047e88cb73e351e2bf59c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.2 K 2024-11-16T16:14:38,303 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d91b66b70064cdbb4e766e58b380fcb, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674647 2024-11-16T16:14:38,304 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 41ad33ebd4c047e88cb73e351e2bf59c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674647 2024-11-16T16:14:38,305 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27e2d236e92c4c4ca1f76f04f1b1e0cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731773675817 2024-11-16T16:14:38,305 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f76e0600638342f989571c6feeeecfc9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731773675817 2024-11-16T16:14:38,305 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 51fbfd401b314e56a973f247bca30dc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676992 2024-11-16T16:14:38,305 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03ebb4e35acf4bc68bf3db100bb39684, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676984 2024-11-16T16:14:38,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=30 2024-11-16T16:14:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:38,321 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:14:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:38,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:38,330 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:38,331 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/62e8fb055a774da09df042cfc3013c99 is 50, key is test_row_0/A:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:38,346 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:38,347 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4011abeca63f4c7492350fe7b72aa601 is 50, key is test_row_0/B:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:38,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9b1e584e78a8465ab59b2e2bb15b7d5c is 50, key is test_row_0/A:col10/1731773677382/Put/seqid=0 2024-11-16T16:14:38,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741947_1123 (size=13561) 2024-11-16T16:14:38,389 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/62e8fb055a774da09df042cfc3013c99 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/62e8fb055a774da09df042cfc3013c99 2024-11-16T16:14:38,403 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into 62e8fb055a774da09df042cfc3013c99(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:38,403 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:38,403 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773678300; duration=0sec 2024-11-16T16:14:38,403 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:38,403 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:38,403 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:38,405 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:38,405 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:38,405 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:38,405 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4a62dfbe60e442aea2d3995e99ae6bf8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.1 K 2024-11-16T16:14:38,407 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a62dfbe60e442aea2d3995e99ae6bf8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1731773674647 2024-11-16T16:14:38,407 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d61a157f81488fae45a4f38a6b9f15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731773675817 2024-11-16T16:14:38,408 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7152475810fc494bacd3a34b75362019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676992 2024-11-16T16:14:38,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741948_1124 (size=13561) 2024-11-16T16:14:38,424 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4011abeca63f4c7492350fe7b72aa601 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4011abeca63f4c7492350fe7b72aa601 2024-11-16T16:14:38,432 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 4011abeca63f4c7492350fe7b72aa601(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:38,432 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:38,432 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773678300; duration=0sec 2024-11-16T16:14:38,432 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:38,432 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:38,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741949_1125 (size=12301) 2024-11-16T16:14:38,435 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9b1e584e78a8465ab59b2e2bb15b7d5c 2024-11-16T16:14:38,435 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#113 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:38,436 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/d2867af3957c41b8b1c6e7adad6127ba is 50, key is test_row_0/C:col10/1731773677317/Put/seqid=0 2024-11-16T16:14:38,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741950_1126 (size=13527) 2024-11-16T16:14:38,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f2352ad049634a44a24048e90329acda is 50, key is test_row_0/B:col10/1731773677382/Put/seqid=0 2024-11-16T16:14:38,470 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/d2867af3957c41b8b1c6e7adad6127ba as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/d2867af3957c41b8b1c6e7adad6127ba 2024-11-16T16:14:38,480 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into d2867af3957c41b8b1c6e7adad6127ba(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:38,480 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:38,481 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773678300; duration=0sec 2024-11-16T16:14:38,481 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:38,481 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:38,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T16:14:38,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741951_1127 (size=12301) 2024-11-16T16:14:38,490 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f2352ad049634a44a24048e90329acda 2024-11-16T16:14:38,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45c6cb6a5e5e446689324d7383d05d88 is 50, key is test_row_0/C:col10/1731773677382/Put/seqid=0 2024-11-16T16:14:38,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741952_1128 (size=12301) 2024-11-16T16:14:38,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:38,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:38,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773738557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773738561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773738665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773738665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T16:14:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773738875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773738875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:38,919 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45c6cb6a5e5e446689324d7383d05d88 2024-11-16T16:14:38,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/9b1e584e78a8465ab59b2e2bb15b7d5c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c 2024-11-16T16:14:38,929 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c, entries=150, sequenceid=523, filesize=12.0 K 2024-11-16T16:14:38,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/f2352ad049634a44a24048e90329acda as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda 2024-11-16T16:14:38,935 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda, entries=150, sequenceid=523, filesize=12.0 K 2024-11-16T16:14:38,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/45c6cb6a5e5e446689324d7383d05d88 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88 2024-11-16T16:14:38,942 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88, entries=150, sequenceid=523, filesize=12.0 K 2024-11-16T16:14:38,945 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 8161ddf0266b9f8b7a8b291995378f22 in 624ms, sequenceid=523, compaction requested=false 2024-11-16T16:14:38,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:38,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:38,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-16T16:14:38,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-11-16T16:14:38,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-16T16:14:38,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 778 msec 2024-11-16T16:14:38,956 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=29, table=TestAcidGuarantees in 789 msec 2024-11-16T16:14:39,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:39,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:39,147 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:39,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a2ac271bfe4a491bb7c8c9c0d2a62738 is 50, key is test_row_0/A:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:39,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741953_1129 (size=14741) 2024-11-16T16:14:39,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a2ac271bfe4a491bb7c8c9c0d2a62738 2024-11-16T16:14:39,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8063444d5c444c6f9e6e6dfd08c163d2 is 50, key is test_row_0/B:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:39,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741954_1130 (size=12301) 2024-11-16T16:14:39,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773739190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773739191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773739192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-16T16:14:39,306 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:39,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees 2024-11-16T16:14:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:39,308 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:39,309 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=31, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,309 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773739305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773739305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773739306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:39,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T16:14:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773739514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773739514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773739515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8063444d5c444c6f9e6e6dfd08c163d2 2024-11-16T16:14:39,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/1b2b9192db2f4ab2be0a32d97e0957f8 is 50, key is test_row_0/C:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:39,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741955_1131 (size=12301) 2024-11-16T16:14:39,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T16:14:39,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:39,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:39,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T16:14:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60752 deadline: 1731773739825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60728 deadline: 1731773739825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60694 deadline: 1731773739825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:14:39,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T16:14:39,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:39,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:39,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] handler.RSProcedureHandler(58): pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=32 java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=32 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:39,982 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-16T16:14:39,982 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:39,986 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,986 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-16T16:14:39,986 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:39,986 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,987 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,987 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:39,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:39,989 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=543 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/1b2b9192db2f4ab2be0a32d97e0957f8 2024-11-16T16:14:39,991 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-16T16:14:39,991 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:39,991 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,991 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:39,994 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-16T16:14:39,994 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:39,994 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,995 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:39,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:39,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/a2ac271bfe4a491bb7c8c9c0d2a62738 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738 2024-11-16T16:14:40,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738, entries=200, sequenceid=543, filesize=14.4 K 2024-11-16T16:14:40,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/8063444d5c444c6f9e6e6dfd08c163d2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2 2024-11-16T16:14:40,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2, entries=150, sequenceid=543, filesize=12.0 K 2024-11-16T16:14:40,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/1b2b9192db2f4ab2be0a32d97e0957f8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8 2024-11-16T16:14:40,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8, entries=150, sequenceid=543, filesize=12.0 K 2024-11-16T16:14:40,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 8161ddf0266b9f8b7a8b291995378f22 in 868ms, sequenceid=543, compaction requested=true 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:40,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8161ddf0266b9f8b7a8b291995378f22:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:40,014 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:40,014 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:40,015 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38163 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:40,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40603 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:40,015 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/B is initiating minor compaction (all files) 2024-11-16T16:14:40,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/A is initiating minor compaction (all files) 2024-11-16T16:14:40,016 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/B in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:40,016 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/A in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:40,016 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/62e8fb055a774da09df042cfc3013c99, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=39.7 K 2024-11-16T16:14:40,016 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4011abeca63f4c7492350fe7b72aa601, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.3 K 2024-11-16T16:14:40,016 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4011abeca63f4c7492350fe7b72aa601, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676992 2024-11-16T16:14:40,016 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62e8fb055a774da09df042cfc3013c99, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676992 2024-11-16T16:14:40,016 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b1e584e78a8465ab59b2e2bb15b7d5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731773677378 2024-11-16T16:14:40,016 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f2352ad049634a44a24048e90329acda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731773677378 2024-11-16T16:14:40,017 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2ac271bfe4a491bb7c8c9c0d2a62738, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731773678551 2024-11-16T16:14:40,017 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8063444d5c444c6f9e6e6dfd08c163d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731773678551 2024-11-16T16:14:40,024 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#A#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:40,025 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:40,025 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/aa6f7e3558c44d10826d7868e855972b is 50, key is test_row_0/A:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:40,025 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/0b60d24259764439b980496084d68e7a is 50, key is test_row_0/B:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:40,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741956_1132 (size=13663) 2024-11-16T16:14:40,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741957_1133 (size=13663) 2024-11-16T16:14:40,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=32 2024-11-16T16:14:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:40,074 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T16:14:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:40,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c27ae56efc624d68a3b55b97ab24881b is 50, key is test_row_0/A:col10/1731773679189/Put/seqid=0 2024-11-16T16:14:40,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741958_1134 (size=12301) 2024-11-16T16:14:40,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. as already flushing 2024-11-16T16:14:40,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:40,341 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-16T16:14:40,341 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:40,341 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-16T16:14:40,341 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,341 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-16T16:14:40,341 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:40,341 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:40,341 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,341 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,341 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,342 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:40,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:40,342 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:40,342 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:40,441 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/aa6f7e3558c44d10826d7868e855972b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/aa6f7e3558c44d10826d7868e855972b 2024-11-16T16:14:40,441 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/0b60d24259764439b980496084d68e7a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0b60d24259764439b980496084d68e7a 2024-11-16T16:14:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:40,448 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/A of 8161ddf0266b9f8b7a8b291995378f22 into aa6f7e3558c44d10826d7868e855972b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:40,448 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/B of 8161ddf0266b9f8b7a8b291995378f22 into 0b60d24259764439b980496084d68e7a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:40,448 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/A, priority=13, startTime=1731773680014; duration=0sec 2024-11-16T16:14:40,448 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/B, priority=13, startTime=1731773680014; duration=0sec 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:A 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:40,448 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:B 2024-11-16T16:14:40,449 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:40,450 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:40,450 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 8161ddf0266b9f8b7a8b291995378f22/C is initiating minor compaction (all files) 2024-11-16T16:14:40,450 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8161ddf0266b9f8b7a8b291995378f22/C in TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:40,450 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/d2867af3957c41b8b1c6e7adad6127ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp, totalSize=37.2 K 2024-11-16T16:14:40,450 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2867af3957c41b8b1c6e7adad6127ba, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773676992 2024-11-16T16:14:40,451 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c6cb6a5e5e446689324d7383d05d88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731773677378 2024-11-16T16:14:40,451 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b2b9192db2f4ab2be0a32d97e0957f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=543, earliestPutTs=1731773678551 2024-11-16T16:14:40,460 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8161ddf0266b9f8b7a8b291995378f22#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:40,460 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/980a03c450d143e78ff62af8505633d8 is 50, key is test_row_0/C:col10/1731773678558/Put/seqid=0 2024-11-16T16:14:40,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741959_1135 (size=13629) 2024-11-16T16:14:40,487 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c27ae56efc624d68a3b55b97ab24881b 2024-11-16T16:14:40,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/94a59d45e0d345c7a84f51a6863d366c is 50, key is test_row_0/B:col10/1731773679189/Put/seqid=0 2024-11-16T16:14:40,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741960_1136 (size=12301) 2024-11-16T16:14:40,852 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:14:40,875 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/980a03c450d143e78ff62af8505633d8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/980a03c450d143e78ff62af8505633d8 2024-11-16T16:14:40,881 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8161ddf0266b9f8b7a8b291995378f22/C of 8161ddf0266b9f8b7a8b291995378f22 into 980a03c450d143e78ff62af8505633d8(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:40,881 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:40,881 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., storeName=8161ddf0266b9f8b7a8b291995378f22/C, priority=13, startTime=1731773680014; duration=0sec 2024-11-16T16:14:40,881 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:40,881 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8161ddf0266b9f8b7a8b291995378f22:C 2024-11-16T16:14:40,901 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/94a59d45e0d345c7a84f51a6863d366c 2024-11-16T16:14:40,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e2dab55a8d5e4fdba3630212f9bb6374 is 50, key is test_row_0/C:col10/1731773679189/Put/seqid=0 2024-11-16T16:14:40,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741961_1137 (size=12301) 2024-11-16T16:14:41,318 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=558 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e2dab55a8d5e4fdba3630212f9bb6374 2024-11-16T16:14:41,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/c27ae56efc624d68a3b55b97ab24881b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c27ae56efc624d68a3b55b97ab24881b 2024-11-16T16:14:41,337 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c27ae56efc624d68a3b55b97ab24881b, entries=150, sequenceid=558, filesize=12.0 K 2024-11-16T16:14:41,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/94a59d45e0d345c7a84f51a6863d366c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/94a59d45e0d345c7a84f51a6863d366c 2024-11-16T16:14:41,343 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/94a59d45e0d345c7a84f51a6863d366c, entries=150, sequenceid=558, filesize=12.0 K 2024-11-16T16:14:41,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/e2dab55a8d5e4fdba3630212f9bb6374 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e2dab55a8d5e4fdba3630212f9bb6374 2024-11-16T16:14:41,349 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e2dab55a8d5e4fdba3630212f9bb6374, entries=150, sequenceid=558, filesize=12.0 K 2024-11-16T16:14:41,350 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=20.13 KB/20610 for 8161ddf0266b9f8b7a8b291995378f22 in 1276ms, sequenceid=558, compaction requested=false 2024-11-16T16:14:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.HRegion(2603): Flush status journal for 8161ddf0266b9f8b7a8b291995378f22: 2024-11-16T16:14:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-16T16:14:41,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=32 2024-11-16T16:14:41,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-16T16:14:41,355 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-16T16:14:41,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=31, table=TestAcidGuarantees in 2.0490 sec 2024-11-16T16:14:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-16T16:14:41,456 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:42,300 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-16T16:14:42,300 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:42,300 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,300 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,300 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:42,336 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-16T16:14:42,336 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:14:42,336 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,336 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2748 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2728 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1213 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3632 rows 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1208 2024-11-16T16:14:42,337 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3614 rows 2024-11-16T16:14:42,337 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:14:42,337 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:14:42,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:42,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:14:42,342 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:14:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:42,350 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773682350"}]},"ts":"1731773682350"} 2024-11-16T16:14:42,353 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:14:42,353 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:14:42,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:14:42,359 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, UNASSIGN}] 2024-11-16T16:14:42,360 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, UNASSIGN 2024-11-16T16:14:42,361 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8161ddf0266b9f8b7a8b291995378f22, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:42,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, UNASSIGN because future has completed 2024-11-16T16:14:42,364 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:14:42,365 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:42,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:42,521 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:42,522 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:14:42,522 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 8161ddf0266b9f8b7a8b291995378f22, disabling compactions & flushes 2024-11-16T16:14:42,522 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:42,522 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:42,522 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. after waiting 0 ms 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:42,523 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(2902): Flushing 8161ddf0266b9f8b7a8b291995378f22 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=A 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=B 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8161ddf0266b9f8b7a8b291995378f22, store=C 2024-11-16T16:14:42,523 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:42,527 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d597cedc34b543b7b8a1f5342b9dd4d8 is 50, key is test_row_0/A:col10/1731773680337/Put/seqid=0 2024-11-16T16:14:42,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741962_1138 (size=12301) 2024-11-16T16:14:42,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:42,969 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d597cedc34b543b7b8a1f5342b9dd4d8 2024-11-16T16:14:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:42,978 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4f463453f16248dea09bdc9c549ead5a is 50, key is test_row_0/B:col10/1731773680337/Put/seqid=0 2024-11-16T16:14:42,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741963_1139 (size=12301) 2024-11-16T16:14:43,383 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4f463453f16248dea09bdc9c549ead5a 2024-11-16T16:14:43,393 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/81c4c1e98abe459f852c14756e8c8087 is 50, key is test_row_0/C:col10/1731773680337/Put/seqid=0 2024-11-16T16:14:43,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741964_1140 (size=12301) 2024-11-16T16:14:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:43,799 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/81c4c1e98abe459f852c14756e8c8087 2024-11-16T16:14:43,804 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/A/d597cedc34b543b7b8a1f5342b9dd4d8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d597cedc34b543b7b8a1f5342b9dd4d8 2024-11-16T16:14:43,809 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d597cedc34b543b7b8a1f5342b9dd4d8, entries=150, sequenceid=569, filesize=12.0 K 2024-11-16T16:14:43,810 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/B/4f463453f16248dea09bdc9c549ead5a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4f463453f16248dea09bdc9c549ead5a 2024-11-16T16:14:43,815 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4f463453f16248dea09bdc9c549ead5a, entries=150, sequenceid=569, filesize=12.0 K 2024-11-16T16:14:43,816 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/.tmp/C/81c4c1e98abe459f852c14756e8c8087 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/81c4c1e98abe459f852c14756e8c8087 2024-11-16T16:14:43,820 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/81c4c1e98abe459f852c14756e8c8087, entries=150, sequenceid=569, filesize=12.0 K 2024-11-16T16:14:43,821 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8161ddf0266b9f8b7a8b291995378f22 in 1298ms, sequenceid=569, compaction requested=true 2024-11-16T16:14:43,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6aecfffa99fb4fd399009ab463586886, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/cb9509c7689146dd8380ab1b54e9b116, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/e5778a7fe32440908e22cd3627335be8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/4ce1061f73dc421188c4e0c69f704179, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/548fbda844f24e8da8329b22234177b5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/3f5a4df792fe4327855d1e24009ba821, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/47eed913216e42f6856c00ee67a7e1a5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bf95d6177ccd491792e80a2da4b53379, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9082add2968d43499b77fcc7a19b6762, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5d91b66b70064cdbb4e766e58b380fcb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/62e8fb055a774da09df042cfc3013c99, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738] to archive 2024-11-16T16:14:43,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:14:43,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/0704695b0b99428489e033a87276a05a 2024-11-16T16:14:43,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/44f1e01a7fd246a3ac6e955557ef640f 2024-11-16T16:14:43,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6aecfffa99fb4fd399009ab463586886 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6aecfffa99fb4fd399009ab463586886 2024-11-16T16:14:43,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6c4fc922a040449ba883f89ee6aaf5e3 2024-11-16T16:14:43,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/1a8e09b9978e47bda389f0207f47d885 2024-11-16T16:14:43,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9920660515d64d34bc60ab4e7f3e06e1 2024-11-16T16:14:43,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/cb9509c7689146dd8380ab1b54e9b116 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/cb9509c7689146dd8380ab1b54e9b116 2024-11-16T16:14:43,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/b3588ba2b5ad48198ab7cef10f8cae70 2024-11-16T16:14:43,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a1f68e8fd6704579a1aa12f252b68128 2024-11-16T16:14:43,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/e5778a7fe32440908e22cd3627335be8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/e5778a7fe32440908e22cd3627335be8 2024-11-16T16:14:43,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ad6fc873404c4d40ac8417f181cbe829 2024-11-16T16:14:43,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/7979293280814b91879c276ebffc8449 2024-11-16T16:14:43,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/4ce1061f73dc421188c4e0c69f704179 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/4ce1061f73dc421188c4e0c69f704179 2024-11-16T16:14:43,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/13ae1defd3b84427872f3b54750936f9 2024-11-16T16:14:43,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/90c77010d0b74a0588df54e4ea8e1fbe 2024-11-16T16:14:43,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bbb5bcfdfc564f049c1df5a2a10d1f3d 2024-11-16T16:14:43,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/548fbda844f24e8da8329b22234177b5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/548fbda844f24e8da8329b22234177b5 2024-11-16T16:14:43,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c9937e6608ac4317bee6667d962af02b 2024-11-16T16:14:43,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/3f5a4df792fe4327855d1e24009ba821 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/3f5a4df792fe4327855d1e24009ba821 2024-11-16T16:14:43,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/39c6da80b0674798988df3a778e7c7c3 2024-11-16T16:14:43,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/6de4f77165af422fba42207c20747a3b 2024-11-16T16:14:43,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/47eed913216e42f6856c00ee67a7e1a5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/47eed913216e42f6856c00ee67a7e1a5 2024-11-16T16:14:43,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/2e4f133cfe2e48d680d9eca07c43386d 2024-11-16T16:14:43,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/81e8743a52e54d8997182b88efd656fe 2024-11-16T16:14:43,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d18027d89ae74b69bc5c48d8d1a29446 2024-11-16T16:14:43,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ee0bb4d5b07b4096a65c3a77f8c0533c 2024-11-16T16:14:43,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5decede801914864bfa514fc9adf96e1 2024-11-16T16:14:43,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bf95d6177ccd491792e80a2da4b53379 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/bf95d6177ccd491792e80a2da4b53379 2024-11-16T16:14:43,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/41a70067ce394656a78f42153921701b 2024-11-16T16:14:43,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9082add2968d43499b77fcc7a19b6762 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9082add2968d43499b77fcc7a19b6762 2024-11-16T16:14:43,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/63ed71c4031748579dc66af595b8c048 2024-11-16T16:14:43,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/288422b6d60f4a3e99cff5e288691bac 2024-11-16T16:14:43,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9ffff6b0e0ff492686f9e461ccff322d 2024-11-16T16:14:43,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/ccb0ef8005894c62a12b76c5791bfc0f 2024-11-16T16:14:43,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5d91b66b70064cdbb4e766e58b380fcb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/5d91b66b70064cdbb4e766e58b380fcb 2024-11-16T16:14:43,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/27e2d236e92c4c4ca1f76f04f1b1e0cd 2024-11-16T16:14:43,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/03ebb4e35acf4bc68bf3db100bb39684 2024-11-16T16:14:43,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/62e8fb055a774da09df042cfc3013c99 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/62e8fb055a774da09df042cfc3013c99 2024-11-16T16:14:43,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/9b1e584e78a8465ab59b2e2bb15b7d5c 2024-11-16T16:14:43,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/a2ac271bfe4a491bb7c8c9c0d2a62738 2024-11-16T16:14:43,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b1eea18aaa64495cabaf68702d431036, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cbd9c5c110614317a77211760e8b41d2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6e675ccd57ab491ba3b05cd9bafbfaff, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cfe7f912645840f38268abc9869f5c28, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e096ab71e5424918adc219dbdd12dda5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a246cc2e95174c2197ead8525d516424, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41aa3846591f408f97bf06f13da49230, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/c509f258cdef4a1db39c12dd999c6318, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41ad33ebd4c047e88cb73e351e2bf59c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4011abeca63f4c7492350fe7b72aa601, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2] to archive 2024-11-16T16:14:43,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:14:43,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8036fdc8f423432a9311f88f884d36cc 2024-11-16T16:14:43,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8e93fa080ee540a4b9f57b96bed6543b 2024-11-16T16:14:43,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b1eea18aaa64495cabaf68702d431036 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b1eea18aaa64495cabaf68702d431036 2024-11-16T16:14:43,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/da66575a43174b77aad2557ce2c4647c 2024-11-16T16:14:43,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cb2d77051ce04bb691b389469de786e7 2024-11-16T16:14:43,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cbd9c5c110614317a77211760e8b41d2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cbd9c5c110614317a77211760e8b41d2 2024-11-16T16:14:43,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4526c09f2b9f42fe813737046b9b492c 2024-11-16T16:14:43,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a0cfcbe9d232495ca3bcc53a65fe1bb9 2024-11-16T16:14:43,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6e675ccd57ab491ba3b05cd9bafbfaff to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6e675ccd57ab491ba3b05cd9bafbfaff 2024-11-16T16:14:43,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/39697d7e1973421898f6e619a1e8068b 2024-11-16T16:14:43,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8088388a076448c2ae351c109038e7f5 2024-11-16T16:14:43,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cfe7f912645840f38268abc9869f5c28 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/cfe7f912645840f38268abc9869f5c28 2024-11-16T16:14:43,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ad96960d7fa641d7998c97ff933adfc9 2024-11-16T16:14:43,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/6703975dc9c04b599c56cf01644b3834 2024-11-16T16:14:43,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b935f19de819439c8b413953ba0537e7 2024-11-16T16:14:43,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e096ab71e5424918adc219dbdd12dda5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e096ab71e5424918adc219dbdd12dda5 2024-11-16T16:14:43,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/bad6c420e0b74101bddb3a9fc8aa30b3 2024-11-16T16:14:43,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/b95e88265b24489094e07a030130bb8e 2024-11-16T16:14:43,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a246cc2e95174c2197ead8525d516424 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a246cc2e95174c2197ead8525d516424 2024-11-16T16:14:43,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/a394e09444a54b109b779492fbce4938 2024-11-16T16:14:43,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0c074750bde944858e5c7ced5d6f4fbc 2024-11-16T16:14:43,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8a4d0d8b7b3143fd8f2ff0d23643bd6a 2024-11-16T16:14:43,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/2b2a7303425a40999beee29decab7d11 2024-11-16T16:14:43,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/ffbfdf434a8443cf86cc4797348689f5 2024-11-16T16:14:43,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/609c32eb88e54fe4a2238b632ce5ca07 2024-11-16T16:14:43,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/81279ad07b3e4abaabf57a4f4a8b525a 2024-11-16T16:14:43,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41aa3846591f408f97bf06f13da49230 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41aa3846591f408f97bf06f13da49230 2024-11-16T16:14:43,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/89223bc9eb02405c9ee64e6132b7b862 2024-11-16T16:14:43,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e0909cb71cbf44b1959d920d06382592 2024-11-16T16:14:43,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/c509f258cdef4a1db39c12dd999c6318 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/c509f258cdef4a1db39c12dd999c6318 2024-11-16T16:14:43,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/e5c9c532740f4d94810b060d8071e7a7 2024-11-16T16:14:43,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8802cd4225784d0f99fb9a43ec296153 2024-11-16T16:14:43,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/5d6a3310b81d4d768d337af8ffc43443 2024-11-16T16:14:43,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41ad33ebd4c047e88cb73e351e2bf59c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/41ad33ebd4c047e88cb73e351e2bf59c 2024-11-16T16:14:43,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4cec96196c8c411fb140370597e5dd53 2024-11-16T16:14:43,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f76e0600638342f989571c6feeeecfc9 2024-11-16T16:14:43,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4011abeca63f4c7492350fe7b72aa601 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4011abeca63f4c7492350fe7b72aa601 2024-11-16T16:14:43,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/51fbfd401b314e56a973f247bca30dc1 2024-11-16T16:14:43,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/f2352ad049634a44a24048e90329acda 2024-11-16T16:14:43,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/8063444d5c444c6f9e6e6dfd08c163d2 2024-11-16T16:14:43,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4c0548b6fdd84d7dbdeaed04f43e4a4c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/66aa639e003f4a9cb4fe49291aed1a18, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/96b28b25dc7e499c84c3a61446207b07, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/514e86b528374c9bb9f9a55832e024b1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e1a8a015968a4bd888ad5eba13afa92f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/a2eef14e18cb4d22b8c053d53926fee0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/eff1c7529f5d446987dc9fa26610afb6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0ef1b73106e644b4ada81eb68bf2ac32, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4a62dfbe60e442aea2d3995e99ae6bf8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/d2867af3957c41b8b1c6e7adad6127ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8] to archive 2024-11-16T16:14:43,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:14:43,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/bc7c478a46e548d188612b248e47540e 2024-11-16T16:14:43,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/37226bfdda114828a4686d4fb7b3fa90 2024-11-16T16:14:43,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/db40b69b9c8048caa887c0af03f58222 2024-11-16T16:14:43,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4c0548b6fdd84d7dbdeaed04f43e4a4c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4c0548b6fdd84d7dbdeaed04f43e4a4c 2024-11-16T16:14:43,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6860afd139214baa957b90f5e61d48fc 2024-11-16T16:14:43,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e82bfbe829ca41b2a90b27e35daf778e 2024-11-16T16:14:43,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/50df3a42484f41e48cc30a87966da53a 2024-11-16T16:14:43,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/66aa639e003f4a9cb4fe49291aed1a18 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/66aa639e003f4a9cb4fe49291aed1a18 2024-11-16T16:14:43,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/da207e32d5444fbab2eedfc1873789f3 2024-11-16T16:14:43,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c419ef1a8fbc4f82a7f7746e470713ba 2024-11-16T16:14:43,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/96b28b25dc7e499c84c3a61446207b07 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/96b28b25dc7e499c84c3a61446207b07 2024-11-16T16:14:43,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45f3ef78d7bc48e8a3297e424e2d2db4 2024-11-16T16:14:43,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/674d0047bd1c4ce49c84a9ede6c76988 2024-11-16T16:14:43,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/92051a003a054c2392f7e421e1f71c0a 2024-11-16T16:14:43,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/514e86b528374c9bb9f9a55832e024b1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/514e86b528374c9bb9f9a55832e024b1 2024-11-16T16:14:43,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/49f5a22d9b8c4086b74148d4f9ee86a4 2024-11-16T16:14:43,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4ce94eb06e044d158086db8c89f831dd 2024-11-16T16:14:43,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e1a8a015968a4bd888ad5eba13afa92f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e1a8a015968a4bd888ad5eba13afa92f 2024-11-16T16:14:43,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0398f5971dda45b0b283c63c11ef1a42 2024-11-16T16:14:43,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c52f7f158cef46c7a928a207dc5123d2 2024-11-16T16:14:43,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/a2eef14e18cb4d22b8c053d53926fee0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/a2eef14e18cb4d22b8c053d53926fee0 2024-11-16T16:14:43,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/ea211da2aa6441c587c8665e196103a1 2024-11-16T16:14:43,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/c07e9293e481478eb77411a010a669f6 2024-11-16T16:14:43,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/6900bde1545d4556bc2c609b0a70416b 2024-11-16T16:14:43,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/b46dab7a7d8144ceb0c5b04eedf314b3 2024-11-16T16:14:43,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/eff1c7529f5d446987dc9fa26610afb6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/eff1c7529f5d446987dc9fa26610afb6 2024-11-16T16:14:43,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/58a216c7dd8a4b24841d2d7111c2ab08 2024-11-16T16:14:43,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/94586f4b37404892905f9dad8310ee10 2024-11-16T16:14:43,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0ef1b73106e644b4ada81eb68bf2ac32 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/0ef1b73106e644b4ada81eb68bf2ac32 2024-11-16T16:14:43,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/48046148f6e34be5899ca9437750978b 2024-11-16T16:14:43,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/889d63f511dd45a29ad3a72fae75c9c1 2024-11-16T16:14:43,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45e450f6df93425ea228771af459ff99 2024-11-16T16:14:43,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4a62dfbe60e442aea2d3995e99ae6bf8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/4a62dfbe60e442aea2d3995e99ae6bf8 2024-11-16T16:14:43,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7c6ac7126c6c4270ad6c0f67cc7a4588 2024-11-16T16:14:43,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/f3d61a157f81488fae45a4f38a6b9f15 2024-11-16T16:14:43,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/d2867af3957c41b8b1c6e7adad6127ba to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/d2867af3957c41b8b1c6e7adad6127ba 2024-11-16T16:14:43,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/7152475810fc494bacd3a34b75362019 2024-11-16T16:14:43,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/45c6cb6a5e5e446689324d7383d05d88 2024-11-16T16:14:43,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/1b2b9192db2f4ab2be0a32d97e0957f8 2024-11-16T16:14:43,990 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/recovered.edits/572.seqid, newMaxSeqId=572, maxSeqId=1 2024-11-16T16:14:43,994 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. 2024-11-16T16:14:43,994 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 8161ddf0266b9f8b7a8b291995378f22: Waiting for close lock at 1731773682522Running coprocessor pre-close hooks at 1731773682522Disabling compacts and flushes for region at 1731773682522Disabling writes for close at 1731773682523 (+1 ms)Obtaining lock to block concurrent updates at 1731773682523Preparing flush snapshotting stores in 8161ddf0266b9f8b7a8b291995378f22 at 1731773682523Finished memstore snapshotting TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731773682523Flushing stores of TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22. at 1731773682524 (+1 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/A: creating writer at 1731773682524Flushing 8161ddf0266b9f8b7a8b291995378f22/A: appending metadata at 1731773682527 (+3 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/A: closing flushed file at 1731773682527Flushing 8161ddf0266b9f8b7a8b291995378f22/B: creating writer at 1731773682974 (+447 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/B: appending metadata at 1731773682978 (+4 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/B: closing flushed file at 1731773682978Flushing 8161ddf0266b9f8b7a8b291995378f22/C: creating writer at 1731773683389 (+411 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/C: appending metadata at 1731773683392 (+3 ms)Flushing 8161ddf0266b9f8b7a8b291995378f22/C: closing flushed file at 1731773683392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b4188ba: reopening flushed file at 1731773683803 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36189a3: reopening flushed file at 1731773683809 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c090d99: reopening flushed file at 1731773683815 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 8161ddf0266b9f8b7a8b291995378f22 in 1298ms, sequenceid=569, compaction requested=true at 1731773683821 (+6 ms)Writing region close event to WAL at 1731773683985 (+164 ms)Running coprocessor post-close hooks at 1731773683993 (+8 ms)Closed at 1731773683994 (+1 ms) 2024-11-16T16:14:43,997 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:43,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8161ddf0266b9f8b7a8b291995378f22, regionState=CLOSED 2024-11-16T16:14:43,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:14:44,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-16T16:14:44,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 8161ddf0266b9f8b7a8b291995378f22, server=303fe01965e8,37355,1731773652566 in 1.6360 sec 2024-11-16T16:14:44,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-16T16:14:44,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8161ddf0266b9f8b7a8b291995378f22, UNASSIGN in 1.6440 sec 2024-11-16T16:14:44,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-16T16:14:44,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6510 sec 2024-11-16T16:14:44,007 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773684007"}]},"ts":"1731773684007"} 2024-11-16T16:14:44,009 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:14:44,009 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:14:44,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6660 sec 2024-11-16T16:14:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-16T16:14:44,497 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:44,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:14:44,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,510 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=37, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-16T16:14:44,512 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=37, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,519 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:44,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/recovered.edits] 2024-11-16T16:14:44,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/aa6f7e3558c44d10826d7868e855972b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/aa6f7e3558c44d10826d7868e855972b 2024-11-16T16:14:44,528 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c27ae56efc624d68a3b55b97ab24881b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/c27ae56efc624d68a3b55b97ab24881b 2024-11-16T16:14:44,530 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d597cedc34b543b7b8a1f5342b9dd4d8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/A/d597cedc34b543b7b8a1f5342b9dd4d8 2024-11-16T16:14:44,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0b60d24259764439b980496084d68e7a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/0b60d24259764439b980496084d68e7a 2024-11-16T16:14:44,533 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4f463453f16248dea09bdc9c549ead5a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/4f463453f16248dea09bdc9c549ead5a 2024-11-16T16:14:44,534 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/94a59d45e0d345c7a84f51a6863d366c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/B/94a59d45e0d345c7a84f51a6863d366c 2024-11-16T16:14:44,536 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/81c4c1e98abe459f852c14756e8c8087 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/81c4c1e98abe459f852c14756e8c8087 2024-11-16T16:14:44,537 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/980a03c450d143e78ff62af8505633d8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/980a03c450d143e78ff62af8505633d8 2024-11-16T16:14:44,538 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e2dab55a8d5e4fdba3630212f9bb6374 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/C/e2dab55a8d5e4fdba3630212f9bb6374 2024-11-16T16:14:44,541 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/recovered.edits/572.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22/recovered.edits/572.seqid 2024-11-16T16:14:44,541 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/8161ddf0266b9f8b7a8b291995378f22 2024-11-16T16:14:44,541 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:14:44,547 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=37, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-16T16:14:44,557 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:14:44,589 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:14:44,590 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=37, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,590 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:14:44,591 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773684591"}]},"ts":"9223372036854775807"} 2024-11-16T16:14:44,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:14:44,595 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8161ddf0266b9f8b7a8b291995378f22, NAME => 'TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:14:44,595 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:14:44,595 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773684595"}]},"ts":"9223372036854775807"} 2024-11-16T16:14:44,598 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:14:44,599 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=37, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 93 msec 2024-11-16T16:14:44,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-16T16:14:44,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:14:44,616 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:44,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-500171457_22 at /127.0.0.1:34516 [Waiting for operation #186] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/303fe01965e8:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1459812294_22 at /127.0.0.1:56578 [Waiting for operation #297] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;303fe01965e8:37355-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1459812294_22 at /127.0.0.1:56462 [Waiting for operation #314] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=365 (was 226) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4045 (was 4607) 2024-11-16T16:14:44,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=4045 2024-11-16T16:14:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:14:44,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:14:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:44,644 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:14:44,644 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:44,644 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 38 2024-11-16T16:14:44,645 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:14:44,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:44,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741965_1141 (size=1061) 2024-11-16T16:14:44,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:45,054 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:14:45,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741966_1142 (size=53) 2024-11-16T16:14:45,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3ed8b6f0ed534a06346c44e18fa12f8f, disabling compactions & flushes 2024-11-16T16:14:45,463 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. after waiting 0 ms 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,463 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,463 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3ed8b6f0ed534a06346c44e18fa12f8f: Waiting for close lock at 1731773685463Disabling compacts and flushes for region at 1731773685463Disabling writes for close at 1731773685463Writing region close event to WAL at 1731773685463Closed at 1731773685463 2024-11-16T16:14:45,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:14:45,465 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773685464"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773685464"}]},"ts":"1731773685464"} 2024-11-16T16:14:45,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:14:45,468 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:14:45,468 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773685468"}]},"ts":"1731773685468"} 2024-11-16T16:14:45,470 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:14:45,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, ASSIGN}] 2024-11-16T16:14:45,471 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, ASSIGN 2024-11-16T16:14:45,472 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:14:45,623 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:45,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, ASSIGN because future has completed 2024-11-16T16:14:45,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:45,781 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,781 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:14:45,781 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,781 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:45,782 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,782 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,783 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,784 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:45,785 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName A 2024-11-16T16:14:45,785 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:45,786 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:45,786 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,787 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:45,787 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName B 2024-11-16T16:14:45,787 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:45,787 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:45,788 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,788 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:45,789 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName C 2024-11-16T16:14:45,789 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:45,789 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:45,789 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,789 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,790 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,790 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,791 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,791 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,792 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:14:45,793 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,795 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:14:45,795 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 3ed8b6f0ed534a06346c44e18fa12f8f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73916486, jitterRate=0.10144147276878357}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:14:45,795 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:45,797 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 3ed8b6f0ed534a06346c44e18fa12f8f: Running coprocessor pre-open hook at 1731773685782Writing region info on filesystem at 1731773685782Initializing all the Stores at 1731773685783 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773685783Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773685783Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773685783Cleaning up temporary data from old regions at 1731773685791 (+8 ms)Running coprocessor post-open hooks at 1731773685795 (+4 ms)Region opened successfully at 1731773685797 (+2 ms) 2024-11-16T16:14:45,798 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., pid=40, masterSystemTime=1731773685777 2024-11-16T16:14:45,799 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,799 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:45,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:45,801 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:14:45,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-16T16:14:45,804 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 in 177 msec 2024-11-16T16:14:45,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-16T16:14:45,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, ASSIGN in 334 msec 2024-11-16T16:14:45,807 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:14:45,808 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773685807"}]},"ts":"1731773685807"} 2024-11-16T16:14:45,809 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:14:45,810 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=38, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:14:45,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1680 sec 2024-11-16T16:14:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-16T16:14:46,795 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:46,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79527845, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:46,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:46,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:46,799 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743a41ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:46,802 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:46,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf8b3a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:46,803 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:46,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:46,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:46,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:46,808 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:14:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:14:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@ad97762 2024-11-16T16:14:46,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:14:46,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:14:46,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:14:46,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:14:46,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T16:14:46,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741967_1143 (size=1097) 2024-11-16T16:14:47,241 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T16:14:47,241 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T16:14:47,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:14:47,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE}] 2024-11-16T16:14:47,254 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE 2024-11-16T16:14:47,255 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:47,256 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE because future has completed 2024-11-16T16:14:47,257 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:14:47,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:47,411 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(122): Close 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,411 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:14:47,411 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1722): Closing 3ed8b6f0ed534a06346c44e18fa12f8f, disabling compactions & flushes 2024-11-16T16:14:47,411 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,412 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,412 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. after waiting 0 ms 2024-11-16T16:14:47,412 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,422 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T16:14:47,423 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,423 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegion(1676): Region close journal for 3ed8b6f0ed534a06346c44e18fa12f8f: Waiting for close lock at 1731773687411Running coprocessor pre-close hooks at 1731773687411Disabling compacts and flushes for region at 1731773687411Disabling writes for close at 1731773687412 (+1 ms)Writing region close event to WAL at 1731773687413 (+1 ms)Running coprocessor post-close hooks at 1731773687423 (+10 ms)Closed at 1731773687423 2024-11-16T16:14:47,423 WARN [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] regionserver.HRegionServer(3299): Not adding moved region record: 3ed8b6f0ed534a06346c44e18fa12f8f to self. 2024-11-16T16:14:47,426 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=44}] handler.UnassignRegionHandler(157): Closed 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,426 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=CLOSED 2024-11-16T16:14:47,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=43, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:14:47,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-11-16T16:14:47,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 in 172 msec 2024-11-16T16:14:47,432 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE; state=CLOSED, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=true 2024-11-16T16:14:47,583 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:47,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=42, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE because future has completed 2024-11-16T16:14:47,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:14:47,746 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,746 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:14:47,746 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,746 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:14:47,746 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,746 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,750 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,751 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:47,755 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName A 2024-11-16T16:14:47,757 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:47,757 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:47,758 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,758 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:47,758 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName B 2024-11-16T16:14:47,758 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:47,759 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:47,759 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,759 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:14:47,760 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3ed8b6f0ed534a06346c44e18fa12f8f columnFamilyName C 2024-11-16T16:14:47,760 DEBUG [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:47,760 INFO [StoreOpener-3ed8b6f0ed534a06346c44e18fa12f8f-1 {}] regionserver.HStore(327): Store=3ed8b6f0ed534a06346c44e18fa12f8f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:14:47,760 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,760 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,761 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,762 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,763 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,763 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,763 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:14:47,765 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,766 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 3ed8b6f0ed534a06346c44e18fa12f8f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69749405, jitterRate=0.039347127079963684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:14:47,766 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,766 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 3ed8b6f0ed534a06346c44e18fa12f8f: Running coprocessor pre-open hook at 1731773687746Writing region info on filesystem at 1731773687746Initializing all the Stores at 1731773687747 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773687747Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773687748 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773687748Cleaning up temporary data from old regions at 1731773687763 (+15 ms)Running coprocessor post-open hooks at 1731773687766 (+3 ms)Region opened successfully at 1731773687766 2024-11-16T16:14:47,767 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., pid=45, masterSystemTime=1731773687743 2024-11-16T16:14:47,769 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,769 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,769 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=43 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=OPEN, openSeqNum=5, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:14:47,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=43, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:14:47,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=43 2024-11-16T16:14:47,774 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=43, state=SUCCESS, hasLock=false; OpenRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 in 183 msec 2024-11-16T16:14:47,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-16T16:14:47,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, REOPEN/MOVE in 521 msec 2024-11-16T16:14:47,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=41 2024-11-16T16:14:47,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=41, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-11-16T16:14:47,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 959 msec 2024-11-16T16:14:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-11-16T16:14:47,783 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:47,784 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,784 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,784 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,785 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119512ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,787 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7033508b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,789 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63e7d43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,790 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f1036d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51d91e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,792 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,792 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,793 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c175b24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,794 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2463e15b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,795 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b19f019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,798 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7867abdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b75e2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,799 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,799 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,800 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6256fb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,801 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@280d4eb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,803 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72b81c6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,804 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103d3fe1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,806 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a776fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46bfc8e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,809 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67ba5a5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,809 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679bff6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dff183a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:14:47,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:14:47,813 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d959f2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,813 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:14:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:14:47,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:14:47,815 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:14:47,815 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17322130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:14:47,816 DEBUG [Thread-711 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,816 DEBUG [Thread-709 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,816 DEBUG [Thread-720 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,816 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:47,817 DEBUG [Thread-716 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,817 DEBUG [Thread-714 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,817 DEBUG [Thread-703 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,818 DEBUG [Thread-714 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,818 DEBUG [Thread-714 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,818 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-16T16:14:47,819 DEBUG [Thread-705 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,820 DEBUG [Thread-707 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,820 DEBUG [Thread-705 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:47,820 DEBUG [Thread-705 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,821 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,821 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:47,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,823 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:47,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,832 DEBUG [Thread-718 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:14:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:14:47,834 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:14:47,835 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:14:47,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,836 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:47,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:14:47,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116455bf6d7020b4f369187b49ad1d62607_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773687834/Put/seqid=0 2024-11-16T16:14:47,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773747887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:47,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:47,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773747888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:47,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773687869, max timestamp=1731773687888Num uniques 859; 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773687869, max timestamp=1731773687888Num uniques 859; 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773687869, max timestamp=1731773687888Num uniques 859; 2024-11-16T16:14:47,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:47,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741968_1144 (size=17034) 2024-11-16T16:14:47,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:47,914 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116455bf6d7020b4f369187b49ad1d62607_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116455bf6d7020b4f369187b49ad1d62607_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:47,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/338a83c138704b08b47eced06b25e419, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:47,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/338a83c138704b08b47eced06b25e419 is 175, key is test_row_0/A:col10/1731773687834/Put/seqid=0 2024-11-16T16:14:47,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:47,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:47,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:47,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:47,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:47,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:47,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741969_1145 (size=48139) 2024-11-16T16:14:47,995 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/338a83c138704b08b47eced06b25e419 2024-11-16T16:14:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773747995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773747996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773747997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773748006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773748007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c5f66c0f4db24ac785a6fd25d42809f5 is 50, key is test_row_0/B:col10/1731773687834/Put/seqid=0 2024-11-16T16:14:48,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741970_1146 (size=12001) 2024-11-16T16:14:48,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773748117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773748126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773748126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,133 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:48,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773748216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773748216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773748325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,327 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773748335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773748345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c5f66c0f4db24ac785a6fd25d42809f5 2024-11-16T16:14:48,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:48,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/8f59d56c7bc040eebefdf73ec63f2864 is 50, key is test_row_0/C:col10/1731773687834/Put/seqid=0 2024-11-16T16:14:48,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741971_1147 (size=12001) 2024-11-16T16:14:48,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/8f59d56c7bc040eebefdf73ec63f2864 2024-11-16T16:14:48,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/338a83c138704b08b47eced06b25e419 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419 2024-11-16T16:14:48,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419, entries=250, sequenceid=15, filesize=47.0 K 2024-11-16T16:14:48,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c5f66c0f4db24ac785a6fd25d42809f5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5 2024-11-16T16:14:48,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T16:14:48,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/8f59d56c7bc040eebefdf73ec63f2864 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864 2024-11-16T16:14:48,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864, entries=150, sequenceid=15, filesize=11.7 K 2024-11-16T16:14:48,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 3ed8b6f0ed534a06346c44e18fa12f8f in 698ms, sequenceid=15, compaction requested=false 2024-11-16T16:14:48,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:48,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:48,544 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:48,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ddf8a6ae1488420a803f6ed2c851ef79_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773688537/Put/seqid=0 2024-11-16T16:14:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773748583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773748583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741972_1148 (size=17034) 2024-11-16T16:14:48,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773748635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773748645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773748655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773748696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773748696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:48,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:48,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773748925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773748925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:48,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:48,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:49,008 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ddf8a6ae1488420a803f6ed2c851ef79_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ddf8a6ae1488420a803f6ed2c851ef79_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:49,009 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/4d994c64c4e64166b0ad9e1dfcb75be6, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:49,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/4d994c64c4e64166b0ad9e1dfcb75be6 is 175, key is test_row_0/A:col10/1731773688537/Put/seqid=0 2024-11-16T16:14:49,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741973_1149 (size=48139) 2024-11-16T16:14:49,025 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/4d994c64c4e64166b0ad9e1dfcb75be6 2024-11-16T16:14:49,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/337001c849cb4b4fa68e1de90430fe62 is 50, key is test_row_0/B:col10/1731773688537/Put/seqid=0 2024-11-16T16:14:49,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741974_1150 (size=12001) 2024-11-16T16:14:49,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773749154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773749155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773749165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773749245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773749245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/337001c849cb4b4fa68e1de90430fe62 2024-11-16T16:14:49,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/fba367d81af14e948ccd4b18a58ae802 is 50, key is test_row_0/C:col10/1731773688537/Put/seqid=0 2024-11-16T16:14:49,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741975_1151 (size=12001) 2024-11-16T16:14:49,529 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:14:49,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773749755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773749755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:49,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:49,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:49,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:49,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/fba367d81af14e948ccd4b18a58ae802 2024-11-16T16:14:49,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/4d994c64c4e64166b0ad9e1dfcb75be6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6 2024-11-16T16:14:49,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6, entries=250, sequenceid=51, filesize=47.0 K 2024-11-16T16:14:49,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/337001c849cb4b4fa68e1de90430fe62 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62 2024-11-16T16:14:49,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T16:14:49,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/fba367d81af14e948ccd4b18a58ae802 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802 2024-11-16T16:14:49,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T16:14:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1349ms, sequenceid=51, compaction requested=false 2024-11-16T16:14:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:49,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:49,990 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:49,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:50,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166ca01d243d5f4e2284fc679718320710_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773688581/Put/seqid=0 2024-11-16T16:14:50,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741976_1152 (size=12154) 2024-11-16T16:14:50,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:50,036 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166ca01d243d5f4e2284fc679718320710_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166ca01d243d5f4e2284fc679718320710_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:50,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/09b8e97302fe4753b6deee8062d1b276, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:50,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/09b8e97302fe4753b6deee8062d1b276 is 175, key is test_row_0/A:col10/1731773688581/Put/seqid=0 2024-11-16T16:14:50,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741977_1153 (size=30955) 2024-11-16T16:14:50,069 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/09b8e97302fe4753b6deee8062d1b276 2024-11-16T16:14:50,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2d2438b76592493194dd5c24f9ad5dfd is 50, key is test_row_0/B:col10/1731773688581/Put/seqid=0 2024-11-16T16:14:50,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741978_1154 (size=12001) 2024-11-16T16:14:50,121 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2d2438b76592493194dd5c24f9ad5dfd 2024-11-16T16:14:50,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5c75fb608726440087e339d3f8ffcd39 is 50, key is test_row_0/C:col10/1731773688581/Put/seqid=0 2024-11-16T16:14:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741979_1155 (size=12001) 2024-11-16T16:14:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:50,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:50,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773690199, max timestamp=1731773690206Num uniques 859; 2024-11-16T16:14:50,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773690199, max timestamp=1731773690206Num uniques 859; 2024-11-16T16:14:50,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:50,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:50,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773690199, max timestamp=1731773690206Num uniques 859; 2024-11-16T16:14:50,212 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773750242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773750250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773750252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,255 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773750355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773750365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773750366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,562 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5c75fb608726440087e339d3f8ffcd39 2024-11-16T16:14:50,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/09b8e97302fe4753b6deee8062d1b276 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276 2024-11-16T16:14:50,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773750565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,571 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276, entries=150, sequenceid=61, filesize=30.2 K 2024-11-16T16:14:50,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2d2438b76592493194dd5c24f9ad5dfd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd 2024-11-16T16:14:50,578 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:14:50,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5c75fb608726440087e339d3f8ffcd39 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39 2024-11-16T16:14:50,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773750574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773750575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,582 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,586 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:14:50,587 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ed8b6f0ed534a06346c44e18fa12f8f in 597ms, sequenceid=61, compaction requested=true 2024-11-16T16:14:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:50,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-16T16:14:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-16T16:14:50,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-16T16:14:50,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7650 sec 2024-11-16T16:14:50,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.7740 sec 2024-11-16T16:14:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:50,771 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:50,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:50,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111623002782d2a84177952df55f7376cd81_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:50,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773750790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741980_1156 (size=14594) 2024-11-16T16:14:50,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:50,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773750793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111623002782d2a84177952df55f7376cd81_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623002782d2a84177952df55f7376cd81_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:50,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/3ac188e6edef48bf92c7da6af1303572, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:50,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/3ac188e6edef48bf92c7da6af1303572 is 175, key is test_row_0/A:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:50,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741981_1157 (size=39549) 2024-11-16T16:14:50,811 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/3ac188e6edef48bf92c7da6af1303572 2024-11-16T16:14:50,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/34abf37793aa4565b64d149413282fc6 is 50, key is test_row_0/B:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:50,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741982_1158 (size=12001) 2024-11-16T16:14:50,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773750874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773750885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773750895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773750905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773750905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773751115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773751115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/34abf37793aa4565b64d149413282fc6 2024-11-16T16:14:51,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5fca36ab18df4a08aa72203c62da869b is 50, key is test_row_0/C:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741983_1159 (size=12001) 2024-11-16T16:14:51,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773751384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773751405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773751405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773751425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773751425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:51,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5fca36ab18df4a08aa72203c62da869b 2024-11-16T16:14:51,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/3ac188e6edef48bf92c7da6af1303572 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572 2024-11-16T16:14:51,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572, entries=200, sequenceid=98, filesize=38.6 K 2024-11-16T16:14:51,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/34abf37793aa4565b64d149413282fc6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6 2024-11-16T16:14:51,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T16:14:51,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/5fca36ab18df4a08aa72203c62da869b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b 2024-11-16T16:14:51,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T16:14:51,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 3ed8b6f0ed534a06346c44e18fa12f8f in 889ms, sequenceid=98, compaction requested=true 2024-11-16T16:14:51,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:51,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:51,660 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:51,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:51,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:51,661 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:51,662 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166782 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:51,662 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:51,662 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:14:51,662 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:14:51,662 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:51,662 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:51,662 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=162.9 K 2024-11-16T16:14:51,662 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=46.9 K 2024-11-16T16:14:51,662 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:51,663 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572] 2024-11-16T16:14:51,663 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f66c0f4db24ac785a6fd25d42809f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731773687831 2024-11-16T16:14:51,663 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 337001c849cb4b4fa68e1de90430fe62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773688003 2024-11-16T16:14:51,664 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2438b76592493194dd5c24f9ad5dfd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773688574 2024-11-16T16:14:51,664 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338a83c138704b08b47eced06b25e419, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731773687830 2024-11-16T16:14:51,664 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 34abf37793aa4565b64d149413282fc6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:51,664 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d994c64c4e64166b0ad9e1dfcb75be6, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773687946 2024-11-16T16:14:51,665 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b8e97302fe4753b6deee8062d1b276, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773688574 2024-11-16T16:14:51,665 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ac188e6edef48bf92c7da6af1303572, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:51,673 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:51,675 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:51,675 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/a34868831eaa48caa998206b39b09e72 is 50, key is test_row_0/B:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:51,676 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116f0b1945dd28d4e02ac896e693a021c5a_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:51,680 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116f0b1945dd28d4e02ac896e693a021c5a_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:51,681 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f0b1945dd28d4e02ac896e693a021c5a_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741984_1160 (size=12139) 2024-11-16T16:14:51,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741985_1161 (size=4469) 2024-11-16T16:14:51,712 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#141 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:51,712 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/76a95fcf14a045ce8c644ed647ef7695 is 175, key is test_row_0/A:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:51,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741986_1162 (size=31093) 2024-11-16T16:14:51,726 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/76a95fcf14a045ce8c644ed647ef7695 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695 2024-11-16T16:14:51,733 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into 76a95fcf14a045ce8c644ed647ef7695(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:51,733 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:51,733 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=12, startTime=1731773691660; duration=0sec 2024-11-16T16:14:51,733 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:51,733 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:14:51,733 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:51,735 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:51,735 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:14:51,736 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:51,736 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=46.9 K 2024-11-16T16:14:51,737 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f59d56c7bc040eebefdf73ec63f2864, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731773687831 2024-11-16T16:14:51,738 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba367d81af14e948ccd4b18a58ae802, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773688003 2024-11-16T16:14:51,738 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c75fb608726440087e339d3f8ffcd39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773688574 2024-11-16T16:14:51,739 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fca36ab18df4a08aa72203c62da869b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:51,759 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#142 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:51,759 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/1893c09b91a04f09834e7ece5e43f7df is 50, key is test_row_0/C:col10/1731773690769/Put/seqid=0 2024-11-16T16:14:51,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741987_1163 (size=12139) 2024-11-16T16:14:51,797 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/1893c09b91a04f09834e7ece5e43f7df as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/1893c09b91a04f09834e7ece5e43f7df 2024-11-16T16:14:51,807 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into 1893c09b91a04f09834e7ece5e43f7df(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:51,807 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:51,807 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=12, startTime=1731773691661; duration=0sec 2024-11-16T16:14:51,807 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:51,807 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:14:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:51,938 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:51,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:51,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a38d0e5bda3243b7881f9f7a2a13184c_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773691936/Put/seqid=0 2024-11-16T16:14:51,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741988_1164 (size=14594) 2024-11-16T16:14:51,969 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:51,974 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116a38d0e5bda3243b7881f9f7a2a13184c_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a38d0e5bda3243b7881f9f7a2a13184c_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:51,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-16T16:14:51,975 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:51,976 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/932f1bfce414433588c940332c8b329f, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:51,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:51,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/932f1bfce414433588c940332c8b329f is 175, key is test_row_0/A:col10/1731773691936/Put/seqid=0 2024-11-16T16:14:51,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-16T16:14:51,980 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:51,981 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:51,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:51,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:51,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741989_1165 (size=39549) 2024-11-16T16:14:52,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773691976, max timestamp=1731773691997Num uniques 859; 2024-11-16T16:14:52,000 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:52,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773691976, max timestamp=1731773691997Num uniques 859; 2024-11-16T16:14:52,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773691979, max timestamp=1731773691997Num uniques 859; 2024-11-16T16:14:52,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773752053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773752051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:52,094 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/a34868831eaa48caa998206b39b09e72 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/a34868831eaa48caa998206b39b09e72 2024-11-16T16:14:52,101 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into a34868831eaa48caa998206b39b09e72(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:52,101 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:52,101 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=12, startTime=1731773691660; duration=0sec 2024-11-16T16:14:52,101 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:52,101 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:14:52,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773752165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773752165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:52,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773752374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773752375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/932f1bfce414433588c940332c8b329f 2024-11-16T16:14:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773752397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,405 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/556dead5271741878af0799901170739 is 50, key is test_row_0/B:col10/1731773691936/Put/seqid=0 2024-11-16T16:14:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741990_1166 (size=12001) 2024-11-16T16:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773752425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773752425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,426 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,426 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773752684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773752684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:52,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/556dead5271741878af0799901170739 2024-11-16T16:14:52,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/039c745877984048b63f70cd03227c9b is 50, key is test_row_0/C:col10/1731773691936/Put/seqid=0 2024-11-16T16:14:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741991_1167 (size=12001) 2024-11-16T16:14:52,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:52,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:52,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:53,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773753194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773753195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:53,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:53,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:53,239 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/039c745877984048b63f70cd03227c9b 2024-11-16T16:14:53,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/932f1bfce414433588c940332c8b329f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f 2024-11-16T16:14:53,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f, entries=200, sequenceid=110, filesize=38.6 K 2024-11-16T16:14:53,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/556dead5271741878af0799901170739 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739 2024-11-16T16:14:53,254 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739, entries=150, sequenceid=110, filesize=11.7 K 2024-11-16T16:14:53,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/039c745877984048b63f70cd03227c9b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b 2024-11-16T16:14:53,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b, entries=150, sequenceid=110, filesize=11.7 K 2024-11-16T16:14:53,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1324ms, sequenceid=110, compaction requested=false 2024-11-16T16:14:53,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:53,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-16T16:14:53,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:53,362 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:53,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:53,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e559c106a5e84e64abc4dd3105f01959_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773692051/Put/seqid=0 2024-11-16T16:14:53,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741992_1168 (size=12304) 2024-11-16T16:14:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:53,385 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e559c106a5e84e64abc4dd3105f01959_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e559c106a5e84e64abc4dd3105f01959_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:53,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0a8593b94240490dbab88249898e7740, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0a8593b94240490dbab88249898e7740 is 175, key is test_row_0/A:col10/1731773692051/Put/seqid=0 2024-11-16T16:14:53,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741993_1169 (size=31105) 2024-11-16T16:14:53,394 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0a8593b94240490dbab88249898e7740 2024-11-16T16:14:53,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c0c9ab9b448649908b0563b969f8d066 is 50, key is test_row_0/B:col10/1731773692051/Put/seqid=0 2024-11-16T16:14:53,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741994_1170 (size=12151) 2024-11-16T16:14:53,811 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c0c9ab9b448649908b0563b969f8d066 2024-11-16T16:14:53,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/55029b05bc924f40b3fd0a8adf3b01ab is 50, key is test_row_0/C:col10/1731773692051/Put/seqid=0 2024-11-16T16:14:53,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741995_1171 (size=12151) 2024-11-16T16:14:54,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:54,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:54,226 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/55029b05bc924f40b3fd0a8adf3b01ab 2024-11-16T16:14:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0a8593b94240490dbab88249898e7740 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740 2024-11-16T16:14:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773754225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773754231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,232 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,235 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740, entries=150, sequenceid=146, filesize=30.4 K 2024-11-16T16:14:54,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c0c9ab9b448649908b0563b969f8d066 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066 2024-11-16T16:14:54,240 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T16:14:54,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/55029b05bc924f40b3fd0a8adf3b01ab as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab 2024-11-16T16:14:54,247 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab, entries=150, sequenceid=146, filesize=11.9 K 2024-11-16T16:14:54,248 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 3ed8b6f0ed534a06346c44e18fa12f8f in 886ms, sequenceid=146, compaction requested=true 2024-11-16T16:14:54,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:54,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:54,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-16T16:14:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-16T16:14:54,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-16T16:14:54,251 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2680 sec 2024-11-16T16:14:54,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2760 sec 2024-11-16T16:14:54,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:54,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:54,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:54,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111685a88180c4174681b5489e622268a31b_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:54,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741996_1172 (size=14794) 2024-11-16T16:14:54,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773754379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773694366, max timestamp=1731773694383Num uniques 859; 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773694366, max timestamp=1731773694383Num uniques 859; 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773694366, max timestamp=1731773694383Num uniques 859; 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:14:54,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773754433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773754439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773754445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,446 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773754455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,463 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773754495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773754545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773754554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773754705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,748 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:54,752 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111685a88180c4174681b5489e622268a31b_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685a88180c4174681b5489e622268a31b_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:54,753 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0268f04d24ea4635a09d554a619c83d8, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:54,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0268f04d24ea4635a09d554a619c83d8 is 175, key is test_row_0/A:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:54,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773754755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741997_1173 (size=39749) 2024-11-16T16:14:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,757 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:54,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0268f04d24ea4635a09d554a619c83d8 2024-11-16T16:14:54,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7f7954f45a0a40529641d95c507a32b4 is 50, key is test_row_0/B:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741998_1174 (size=12151) 2024-11-16T16:14:54,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773754775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:54,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773755014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773755065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773755085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7f7954f45a0a40529641d95c507a32b4 2024-11-16T16:14:55,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/de424e2670d14a2584749e0158e3fb6b is 50, key is test_row_0/C:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:55,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741999_1175 (size=12151) 2024-11-16T16:14:55,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/de424e2670d14a2584749e0158e3fb6b 2024-11-16T16:14:55,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/0268f04d24ea4635a09d554a619c83d8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8 2024-11-16T16:14:55,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8, entries=200, sequenceid=157, filesize=38.8 K 2024-11-16T16:14:55,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7f7954f45a0a40529641d95c507a32b4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4 2024-11-16T16:14:55,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4, entries=150, sequenceid=157, filesize=11.9 K 2024-11-16T16:14:55,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/de424e2670d14a2584749e0158e3fb6b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b 2024-11-16T16:14:55,230 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b, entries=150, sequenceid=157, filesize=11.9 K 2024-11-16T16:14:55,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3ed8b6f0ed534a06346c44e18fa12f8f in 895ms, sequenceid=157, compaction requested=true 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:55,232 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:55,232 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:55,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:55,237 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:55,238 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:14:55,238 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:55,238 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=138.2 K 2024-11-16T16:14:55,238 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:55,238 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8] 2024-11-16T16:14:55,239 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:55,239 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:14:55,239 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:55,239 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/a34868831eaa48caa998206b39b09e72, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=47.3 K 2024-11-16T16:14:55,240 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a34868831eaa48caa998206b39b09e72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:55,240 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a95fcf14a045ce8c644ed647ef7695, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:55,240 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 556dead5271741878af0799901170739, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731773690786 2024-11-16T16:14:55,241 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c9ab9b448649908b0563b969f8d066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731773692039 2024-11-16T16:14:55,241 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 932f1bfce414433588c940332c8b329f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731773690786 2024-11-16T16:14:55,241 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7954f45a0a40529641d95c507a32b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:55,242 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a8593b94240490dbab88249898e7740, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731773692039 2024-11-16T16:14:55,243 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0268f04d24ea4635a09d554a619c83d8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:55,254 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#152 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:55,255 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/84d2a149df6a4233bbe24988f52b2839 is 50, key is test_row_0/B:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:55,261 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:55,266 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111670b8f9b409014b788f250f1b0412003d_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:55,270 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111670b8f9b409014b788f250f1b0412003d_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:55,270 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111670b8f9b409014b788f250f1b0412003d_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:55,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742000_1176 (size=12425) 2024-11-16T16:14:55,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742001_1177 (size=4469) 2024-11-16T16:14:55,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:55,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:55,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e4ccaf2dd46f451a9fcb49e953f84db7_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773694423/Put/seqid=0 2024-11-16T16:14:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742002_1178 (size=14794) 2024-11-16T16:14:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773755569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773755575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773755595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,683 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/84d2a149df6a4233bbe24988f52b2839 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/84d2a149df6a4233bbe24988f52b2839 2024-11-16T16:14:55,685 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#153 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:55,685 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/ba388f9fba1a4d85a0189f6d2f6a9d93 is 175, key is test_row_0/A:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:55,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773755685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742003_1179 (size=31379) 2024-11-16T16:14:55,693 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into 84d2a149df6a4233bbe24988f52b2839(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:55,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:55,693 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=12, startTime=1731773695232; duration=0sec 2024-11-16T16:14:55,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:55,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:14:55,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:14:55,696 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:14:55,696 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:14:55,696 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:55,696 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/1893c09b91a04f09834e7ece5e43f7df, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=47.3 K 2024-11-16T16:14:55,697 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1893c09b91a04f09834e7ece5e43f7df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773690240 2024-11-16T16:14:55,700 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 039c745877984048b63f70cd03227c9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731773690786 2024-11-16T16:14:55,700 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 55029b05bc924f40b3fd0a8adf3b01ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731773692039 2024-11-16T16:14:55,701 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting de424e2670d14a2584749e0158e3fb6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:55,703 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/ba388f9fba1a4d85a0189f6d2f6a9d93 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93 2024-11-16T16:14:55,710 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into ba388f9fba1a4d85a0189f6d2f6a9d93(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:55,710 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:55,711 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=12, startTime=1731773695232; duration=0sec 2024-11-16T16:14:55,711 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:55,711 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:14:55,716 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#155 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:55,717 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/d11dbbbc4e9b4096b4ccbdc2d292127d is 50, key is test_row_0/C:col10/1731773694216/Put/seqid=0 2024-11-16T16:14:55,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742004_1180 (size=12425) 2024-11-16T16:14:55,729 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/d11dbbbc4e9b4096b4ccbdc2d292127d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d11dbbbc4e9b4096b4ccbdc2d292127d 2024-11-16T16:14:55,734 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into d11dbbbc4e9b4096b4ccbdc2d292127d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:55,734 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:55,735 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=12, startTime=1731773695232; duration=0sec 2024-11-16T16:14:55,735 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:55,735 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:14:55,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:55,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773755895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:55,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:55,953 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:55,957 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e4ccaf2dd46f451a9fcb49e953f84db7_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4ccaf2dd46f451a9fcb49e953f84db7_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:55,958 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:55,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb is 175, key is test_row_0/A:col10/1731773694423/Put/seqid=0 2024-11-16T16:14:55,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742005_1181 (size=39749) 2024-11-16T16:14:55,969 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb 2024-11-16T16:14:55,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/01fadb0783b34305a774ab97f8207a59 is 50, key is test_row_0/B:col10/1731773694423/Put/seqid=0 2024-11-16T16:14:55,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742006_1182 (size=12151) 2024-11-16T16:14:55,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/01fadb0783b34305a774ab97f8207a59 2024-11-16T16:14:55,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d is 50, key is test_row_0/C:col10/1731773694423/Put/seqid=0 2024-11-16T16:14:56,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742007_1183 (size=12151) 2024-11-16T16:14:56,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d 2024-11-16T16:14:56,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb 2024-11-16T16:14:56,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb, entries=200, sequenceid=192, filesize=38.8 K 2024-11-16T16:14:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/01fadb0783b34305a774ab97f8207a59 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59 2024-11-16T16:14:56,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T16:14:56,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d 2024-11-16T16:14:56,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-16T16:14:56,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 3ed8b6f0ed534a06346c44e18fa12f8f in 507ms, sequenceid=192, compaction requested=false 2024-11-16T16:14:56,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:56,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-16T16:14:56,136 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:56,136 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-16T16:14:56,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:56,138 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:56,139 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:56,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:56,207 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:56,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:56,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c28257a502bf4653b52d331c90cceccb_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:56,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742008_1184 (size=14794) 2024-11-16T16:14:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:56,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773696249, max timestamp=1731773696279Num uniques 859; 2024-11-16T16:14:56,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:56,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773696249, max timestamp=1731773696279Num uniques 859; 2024-11-16T16:14:56,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:56,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773696249, max timestamp=1731773696279Num uniques 859; 2024-11-16T16:14:56,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:14:56,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T16:14:56,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:56,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773756338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:56,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T16:14:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:56,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773756445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:56,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:56,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773756585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:56,590 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:56,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T16:14:56,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773756607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:56,608 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:56,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:56,634 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c28257a502bf4653b52d331c90cceccb_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c28257a502bf4653b52d331c90cceccb_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:56,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6536206cccb54b538b18c86542c91e7a, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:56,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6536206cccb54b538b18c86542c91e7a is 175, key is test_row_0/A:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:56,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742009_1185 (size=39749) 2024-11-16T16:14:56,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6536206cccb54b538b18c86542c91e7a 2024-11-16T16:14:56,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/acefdcdf29eb43bc8fd9b1541d00478e is 50, key is test_row_0/B:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:56,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742010_1186 (size=12151) 2024-11-16T16:14:56,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773756655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:56,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/acefdcdf29eb43bc8fd9b1541d00478e 2024-11-16T16:14:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:56,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/f4267b0909594f3d897146cfc08eef71 is 50, key is test_row_0/C:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:56,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742011_1187 (size=12151) 2024-11-16T16:14:56,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/f4267b0909594f3d897146cfc08eef71 2024-11-16T16:14:56,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6536206cccb54b538b18c86542c91e7a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a 2024-11-16T16:14:56,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a, entries=200, sequenceid=206, filesize=38.8 K 2024-11-16T16:14:56,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/acefdcdf29eb43bc8fd9b1541d00478e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e 2024-11-16T16:14:56,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-16T16:14:56,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/f4267b0909594f3d897146cfc08eef71 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71 2024-11-16T16:14:56,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71, entries=150, sequenceid=206, filesize=11.9 K 2024-11-16T16:14:56,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3ed8b6f0ed534a06346c44e18fa12f8f in 498ms, sequenceid=206, compaction requested=true 2024-11-16T16:14:56,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:56,706 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:56,706 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:56,706 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:56,707 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:56,708 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:56,708 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:14:56,709 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,709 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=108.3 K 2024-11-16T16:14:56,709 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,709 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a] 2024-11-16T16:14:56,711 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba388f9fba1a4d85a0189f6d2f6a9d93, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:56,712 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0f7380f77fe46ea9e4a6ef88aa3f4fb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731773694423 2024-11-16T16:14:56,712 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:56,712 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:14:56,712 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,712 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/84d2a149df6a4233bbe24988f52b2839, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=35.9 K 2024-11-16T16:14:56,713 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6536206cccb54b538b18c86542c91e7a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695544 2024-11-16T16:14:56,713 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d2a149df6a4233bbe24988f52b2839, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:56,714 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fadb0783b34305a774ab97f8207a59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731773694423 2024-11-16T16:14:56,714 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting acefdcdf29eb43bc8fd9b1541d00478e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695557 2024-11-16T16:14:56,720 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:56,723 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411163e1d16a15da34b9ca5b6849e3153a032_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:56,725 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:56,726 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411163e1d16a15da34b9ca5b6849e3153a032_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:56,726 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/af2114703ad0481bb9841e702f5ca923 is 50, key is test_row_0/B:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:56,726 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411163e1d16a15da34b9ca5b6849e3153a032_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:56,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742012_1188 (size=12527) 2024-11-16T16:14:56,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742013_1189 (size=4469) 2024-11-16T16:14:56,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-16T16:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:56,753 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-16T16:14:56,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:56,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:56,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636a721392bc64565b11a4131276e1e94_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773696337/Put/seqid=0 2024-11-16T16:14:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:56,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742014_1190 (size=12304) 2024-11-16T16:14:56,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:57,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:57,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773757017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:57,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773757135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:57,141 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#161 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:57,141 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/b6f597b9095849169bcf02f38e1feda8 is 175, key is test_row_0/A:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:57,142 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/af2114703ad0481bb9841e702f5ca923 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/af2114703ad0481bb9841e702f5ca923 2024-11-16T16:14:57,152 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into af2114703ad0481bb9841e702f5ca923(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:57,152 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:57,152 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=13, startTime=1731773696706; duration=0sec 2024-11-16T16:14:57,152 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:57,152 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:14:57,152 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:57,154 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:57,154 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:14:57,154 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:57,154 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d11dbbbc4e9b4096b4ccbdc2d292127d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=35.9 K 2024-11-16T16:14:57,156 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d11dbbbc4e9b4096b4ccbdc2d292127d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731773694216 2024-11-16T16:14:57,157 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fae6cfd3b2343cdbb0bcd5c2c86f48d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731773694423 2024-11-16T16:14:57,157 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f4267b0909594f3d897146cfc08eef71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695557 2024-11-16T16:14:57,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:57,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742015_1191 (size=31481) 2024-11-16T16:14:57,183 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#164 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:57,183 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/479093064c0545ab8fc57e911e3d490e is 50, key is test_row_0/C:col10/1731773695567/Put/seqid=0 2024-11-16T16:14:57,192 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111636a721392bc64565b11a4131276e1e94_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636a721392bc64565b11a4131276e1e94_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:57,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/38bf796f0bc64b3ba53b99233e8335ff, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:57,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/38bf796f0bc64b3ba53b99233e8335ff is 175, key is test_row_0/A:col10/1731773696337/Put/seqid=0 2024-11-16T16:14:57,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742017_1193 (size=31105) 2024-11-16T16:14:57,222 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/38bf796f0bc64b3ba53b99233e8335ff 2024-11-16T16:14:57,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742016_1192 (size=12527) 2024-11-16T16:14:57,239 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/479093064c0545ab8fc57e911e3d490e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/479093064c0545ab8fc57e911e3d490e 2024-11-16T16:14:57,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/d9fd9acb387741ae82e5b4910c6339fd is 50, key is test_row_0/B:col10/1731773696337/Put/seqid=0 2024-11-16T16:14:57,248 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into 479093064c0545ab8fc57e911e3d490e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:57,248 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:57,249 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=13, startTime=1731773696707; duration=0sec 2024-11-16T16:14:57,249 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:57,249 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:14:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:57,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742018_1194 (size=12151) 2024-11-16T16:14:57,293 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/d9fd9acb387741ae82e5b4910c6339fd 2024-11-16T16:14:57,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/aa3b5457289b43929684f812255ce3fa is 50, key is test_row_0/C:col10/1731773696337/Put/seqid=0 2024-11-16T16:14:57,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773757345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:57,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742019_1195 (size=12151) 2024-11-16T16:14:57,367 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/aa3b5457289b43929684f812255ce3fa 2024-11-16T16:14:57,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/38bf796f0bc64b3ba53b99233e8335ff as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff 2024-11-16T16:14:57,379 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff, entries=150, sequenceid=241, filesize=30.4 K 2024-11-16T16:14:57,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/d9fd9acb387741ae82e5b4910c6339fd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd 2024-11-16T16:14:57,388 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd, entries=150, sequenceid=241, filesize=11.9 K 2024-11-16T16:14:57,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/aa3b5457289b43929684f812255ce3fa as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa 2024-11-16T16:14:57,395 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa, entries=150, sequenceid=241, filesize=11.9 K 2024-11-16T16:14:57,397 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 3ed8b6f0ed534a06346c44e18fa12f8f in 644ms, sequenceid=241, compaction requested=false 2024-11-16T16:14:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:57,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-16T16:14:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-16T16:14:57,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-16T16:14:57,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2600 sec 2024-11-16T16:14:57,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.2650 sec 2024-11-16T16:14:57,586 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/b6f597b9095849169bcf02f38e1feda8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8 2024-11-16T16:14:57,604 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into b6f597b9095849169bcf02f38e1feda8(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:57,605 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:57,605 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=13, startTime=1731773696705; duration=0sec 2024-11-16T16:14:57,605 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:57,606 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:14:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:57,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:57,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:14:57,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167a8024960856489ea809cb88ebe0a912_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742020_1196 (size=14794) 2024-11-16T16:14:57,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:57,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773757748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:57,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773757865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:57,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773758075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:58,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167a8024960856489ea809cb88ebe0a912_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8024960856489ea809cb88ebe0a912_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:58,139 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/1565c5f8c8504eba81de4ef0266ecb96, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:58,139 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/1565c5f8c8504eba81de4ef0266ecb96 is 175, key is test_row_0/A:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:58,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742021_1197 (size=39749) 2024-11-16T16:14:58,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/1565c5f8c8504eba81de4ef0266ecb96 2024-11-16T16:14:58,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/1419f879214d4826b13d5dd65fa58bf5 is 50, key is test_row_0/B:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:58,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742022_1198 (size=12151) 2024-11-16T16:14:58,246 INFO [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-16T16:14:58,246 INFO [master/303fe01965e8:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-16T16:14:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-16T16:14:58,285 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:14:58,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:14:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-16T16:14:58,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:14:58,287 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:14:58,288 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:14:58,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:14:58,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773758385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:14:58,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57440 deadline: 1731773758475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,476 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57512 deadline: 1731773758485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,487 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/1419f879214d4826b13d5dd65fa58bf5 2024-11-16T16:14:58,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/83e1619c476d40e18a299235c7fcff03 is 50, key is test_row_0/C:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:58,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742023_1199 (size=12151) 2024-11-16T16:14:58,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/83e1619c476d40e18a299235c7fcff03 2024-11-16T16:14:58,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/1565c5f8c8504eba81de4ef0266ecb96 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96 2024-11-16T16:14:58,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96, entries=200, sequenceid=256, filesize=38.8 K 2024-11-16T16:14:58,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/1419f879214d4826b13d5dd65fa58bf5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5 2024-11-16T16:14:58,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5, entries=150, sequenceid=256, filesize=11.9 K 2024-11-16T16:14:58,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/83e1619c476d40e18a299235c7fcff03 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03 2024-11-16T16:14:58,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03, entries=150, sequenceid=256, filesize=11.9 K 2024-11-16T16:14:58,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3ed8b6f0ed534a06346c44e18fa12f8f in 929ms, sequenceid=256, compaction requested=true 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:14:58,598 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:58,598 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:14:58,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:58,599 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:58,599 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:58,599 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:14:58,599 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:14:58,599 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,599 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,599 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/af2114703ad0481bb9841e702f5ca923, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.0 K 2024-11-16T16:14:58,599 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=99.9 K 2024-11-16T16:14:58,599 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,599 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96] 2024-11-16T16:14:58,600 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting af2114703ad0481bb9841e702f5ca923, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695557 2024-11-16T16:14:58,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f597b9095849169bcf02f38e1feda8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695557 2024-11-16T16:14:58,600 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d9fd9acb387741ae82e5b4910c6339fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731773696319 2024-11-16T16:14:58,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38bf796f0bc64b3ba53b99233e8335ff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731773696319 2024-11-16T16:14:58,601 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1419f879214d4826b13d5dd65fa58bf5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:14:58,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1565c5f8c8504eba81de4ef0266ecb96, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:14:58,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:14:58,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773697740, max timestamp=1731773698606Num uniques 859; 2024-11-16T16:14:58,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:58,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773697740, max timestamp=1731773698606Num uniques 859; 2024-11-16T16:14:58,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:58,609 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:14:58,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=58.14 KB heapSize=164.17 KB 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:58,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:58,611 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:58,617 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116fdf41e437bfe4b0cb2b5d039ee7717c8_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:58,618 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:58,619 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116fdf41e437bfe4b0cb2b5d039ee7717c8_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:58,619 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/274fd944e89b4f4f9383591b2a1fe2c4 is 50, key is test_row_0/B:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:58,619 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fdf41e437bfe4b0cb2b5d039ee7717c8_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742025_1201 (size=12629) 2024-11-16T16:14:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742024_1200 (size=4469) 2024-11-16T16:14:58,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbc39bc766d0480080f37ec8fa584e7f_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773697740/Put/seqid=0 2024-11-16T16:14:58,639 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#170 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:58,640 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/78b1266ab7ea452682523d34c8eb9287 is 175, key is test_row_0/A:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742026_1202 (size=14994) 2024-11-16T16:14:58,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742027_1203 (size=31583) 2024-11-16T16:14:58,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773698652, max timestamp=1731773698678Num uniques 859; 2024-11-16T16:14:58,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773698652, max timestamp=1731773698678Num uniques 859; 2024-11-16T16:14:58,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731773698652, max timestamp=1731773698678Num uniques 859; 2024-11-16T16:14:58,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:58,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:58,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-16T16:14:58,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773758718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773758719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:58,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773758835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773758835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773758895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:58,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:58,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:58,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:14:59,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/274fd944e89b4f4f9383591b2a1fe2c4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/274fd944e89b4f4f9383591b2a1fe2c4 2024-11-16T16:14:59,047 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:14:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773759045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773759045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,050 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into 274fd944e89b4f4f9383591b2a1fe2c4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:59,050 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:59,050 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=13, startTime=1731773698598; duration=0sec 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,051 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:14:59,051 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:14:59,051 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:14:59,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:59,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:59,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dbc39bc766d0480080f37ec8fa584e7f_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbc39bc766d0480080f37ec8fa584e7f_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:59,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:14:59,056 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:14:59,056 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,056 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/479093064c0545ab8fc57e911e3d490e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.0 K 2024-11-16T16:14:59,058 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 479093064c0545ab8fc57e911e3d490e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731773695557 2024-11-16T16:14:59,058 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/88be1bb998ab4d26b6bfba7dba8a2ed5, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:14:59,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/88be1bb998ab4d26b6bfba7dba8a2ed5 is 175, key is test_row_0/A:col10/1731773697740/Put/seqid=0 2024-11-16T16:14:59,059 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting aa3b5457289b43929684f812255ce3fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731773696319 2024-11-16T16:14:59,061 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/78b1266ab7ea452682523d34c8eb9287 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287 2024-11-16T16:14:59,061 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e1619c476d40e18a299235c7fcff03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:14:59,067 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into 78b1266ab7ea452682523d34c8eb9287(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:59,067 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:59,067 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=13, startTime=1731773698598; duration=0sec 2024-11-16T16:14:59,067 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:59,068 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:14:59,075 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:14:59,075 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/0d6d66a26dba46059f7b11c9a3cad6d3 is 50, key is test_row_0/C:col10/1731773697013/Put/seqid=0 2024-11-16T16:14:59,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742028_1204 (size=39949) 2024-11-16T16:14:59,078 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/88be1bb998ab4d26b6bfba7dba8a2ed5 2024-11-16T16:14:59,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742029_1205 (size=12629) 2024-11-16T16:14:59,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6f48aa997b62459984aa45c9d1f364b2 is 50, key is test_row_0/B:col10/1731773697740/Put/seqid=0 2024-11-16T16:14:59,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742030_1206 (size=12301) 2024-11-16T16:14:59,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773759355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,358 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773759355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:14:59,489 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/0d6d66a26dba46059f7b11c9a3cad6d3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/0d6d66a26dba46059f7b11c9a3cad6d3 2024-11-16T16:14:59,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6f48aa997b62459984aa45c9d1f364b2 2024-11-16T16:14:59,496 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into 0d6d66a26dba46059f7b11c9a3cad6d3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:14:59,496 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:59,496 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=13, startTime=1731773698598; duration=0sec 2024-11-16T16:14:59,496 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:14:59,496 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:14:59,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/25d3428dfe9145719e14ed3ac93bee43 is 50, key is test_row_0/C:col10/1731773697740/Put/seqid=0 2024-11-16T16:14:59,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742031_1207 (size=12301) 2024-11-16T16:14:59,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/25d3428dfe9145719e14ed3ac93bee43 2024-11-16T16:14:59,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/88be1bb998ab4d26b6bfba7dba8a2ed5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5 2024-11-16T16:14:59,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:59,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:59,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:14:59,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5, entries=200, sequenceid=277, filesize=39.0 K 2024-11-16T16:14:59,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6f48aa997b62459984aa45c9d1f364b2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2 2024-11-16T16:14:59,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T16:14:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/25d3428dfe9145719e14ed3ac93bee43 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43 2024-11-16T16:14:59,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43, entries=150, sequenceid=277, filesize=12.0 K 2024-11-16T16:14:59,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.14 KB/59540, heapSize ~164.13 KB/168064, currentSize=127.47 KB/130530 for 3ed8b6f0ed534a06346c44e18fa12f8f in 922ms, sequenceid=277, compaction requested=false 2024-11-16T16:14:59,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:14:59,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-16T16:14:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:14:59,666 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=127.47 KB heapSize=351 KB 2024-11-16T16:14:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:14:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:14:59,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:14:59,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:14:59,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efd208f0edc0411f9e448a5011e7707b_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773698687/Put/seqid=0 2024-11-16T16:14:59,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742032_1208 (size=12454) 2024-11-16T16:14:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:14:59,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:14:59,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773759882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773759883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773759905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,912 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:14:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773759995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773759995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:14:59,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:00,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:00,094 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116efd208f0edc0411f9e448a5011e7707b_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd208f0edc0411f9e448a5011e7707b_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:00,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/450299222db44e2cb45911ac07c81579, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:00,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/450299222db44e2cb45911ac07c81579 is 175, key is test_row_0/A:col10/1731773698687/Put/seqid=0 2024-11-16T16:15:00,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742033_1209 (size=31255) 2024-11-16T16:15:00,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773760205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:00,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773760205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:15:00,500 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/450299222db44e2cb45911ac07c81579 2024-11-16T16:15:00,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c9e0e18988a24b6f815db782d0cc8193 is 50, key is test_row_0/B:col10/1731773698687/Put/seqid=0 2024-11-16T16:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773760515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:00,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773760515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:00,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742034_1210 (size=12301) 2024-11-16T16:15:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:00,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:00,519 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c9e0e18988a24b6f815db782d0cc8193 2024-11-16T16:15:00,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c4fd5b0e985947f3b0f49c8b4a1a5181 is 50, key is test_row_0/C:col10/1731773698687/Put/seqid=0 2024-11-16T16:15:00,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742035_1211 (size=12301) 2024-11-16T16:15:00,933 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c4fd5b0e985947f3b0f49c8b4a1a5181 2024-11-16T16:15:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/450299222db44e2cb45911ac07c81579 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579 2024-11-16T16:15:00,941 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579, entries=150, sequenceid=313, filesize=30.5 K 2024-11-16T16:15:00,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c9e0e18988a24b6f815db782d0cc8193 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193 2024-11-16T16:15:00,946 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T16:15:00,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c4fd5b0e985947f3b0f49c8b4a1a5181 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181 2024-11-16T16:15:00,951 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181, entries=150, sequenceid=313, filesize=12.0 K 2024-11-16T16:15:00,952 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~350.95 KB/359376, currentSize=60.38 KB/61830 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1286ms, sequenceid=313, compaction requested=true 2024-11-16T16:15:00,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:00,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-16T16:15:00,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-16T16:15:00,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-16T16:15:00,956 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6660 sec 2024-11-16T16:15:00,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6700 sec 2024-11-16T16:15:01,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:01,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:01,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:01,037 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164ec3da5c62564489bff6ac3223a5a241_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:01,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742036_1212 (size=14994) 2024-11-16T16:15:01,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773761066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773761073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773761185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773761186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773761395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773761395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,447 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:01,450 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164ec3da5c62564489bff6ac3223a5a241_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ec3da5c62564489bff6ac3223a5a241_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:01,451 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/44989c2f88c04f829f00e3deebc371d4, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:01,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/44989c2f88c04f829f00e3deebc371d4 is 175, key is test_row_0/A:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:01,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742037_1213 (size=39949) 2024-11-16T16:15:01,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773761704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773761704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:01,855 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/44989c2f88c04f829f00e3deebc371d4 2024-11-16T16:15:01,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6210b1552ab546bc8995de9ef52f6223 is 50, key is test_row_0/B:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742038_1214 (size=12301) 2024-11-16T16:15:01,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:01,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773761925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:01,926 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773762215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:02,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773762215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:02,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6210b1552ab546bc8995de9ef52f6223 2024-11-16T16:15:02,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/46625305ada0463f95f6f367c119fe8f is 50, key is test_row_0/C:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:02,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742039_1215 (size=12301) 2024-11-16T16:15:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-16T16:15:02,455 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:02,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-16T16:15:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:02,457 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:02,458 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:02,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:02,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:02,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T16:15:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:02,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:02,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/46625305ada0463f95f6f367c119fe8f 2024-11-16T16:15:02,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/44989c2f88c04f829f00e3deebc371d4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4 2024-11-16T16:15:02,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4, entries=200, sequenceid=327, filesize=39.0 K 2024-11-16T16:15:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/6210b1552ab546bc8995de9ef52f6223 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223 2024-11-16T16:15:02,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223, entries=150, sequenceid=327, filesize=12.0 K 2024-11-16T16:15:02,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/46625305ada0463f95f6f367c119fe8f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f 2024-11-16T16:15:02,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f, entries=150, sequenceid=327, filesize=12.0 K 2024-11-16T16:15:02,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1667ms, sequenceid=327, compaction requested=true 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:02,695 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:02,695 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:02,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:02,697 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142736 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:02,697 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:02,697 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:15:02,697 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:15:02,698 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,698 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,698 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/274fd944e89b4f4f9383591b2a1fe2c4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=48.4 K 2024-11-16T16:15:02,698 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=139.4 K 2024-11-16T16:15:02,698 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,698 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4] 2024-11-16T16:15:02,698 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 274fd944e89b4f4f9383591b2a1fe2c4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:15:02,698 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78b1266ab7ea452682523d34c8eb9287, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:15:02,698 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f48aa997b62459984aa45c9d1f364b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731773697740 2024-11-16T16:15:02,698 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88be1bb998ab4d26b6bfba7dba8a2ed5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731773697740 2024-11-16T16:15:02,699 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c9e0e18988a24b6f815db782d0cc8193, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731773698687 2024-11-16T16:15:02,699 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 450299222db44e2cb45911ac07c81579, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731773698687 2024-11-16T16:15:02,699 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6210b1552ab546bc8995de9ef52f6223, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:02,699 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44989c2f88c04f829f00e3deebc371d4, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:02,714 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:02,716 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#182 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:02,716 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/d5956904f633480999234a5e89bcfde3 is 50, key is test_row_0/B:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:02,725 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111624596dc50c8f4552b04050ede1ae6636_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:02,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742040_1216 (size=12915) 2024-11-16T16:15:02,728 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111624596dc50c8f4552b04050ede1ae6636_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:02,728 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111624596dc50c8f4552b04050ede1ae6636_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:02,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742041_1217 (size=4469) 2024-11-16T16:15:02,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-16T16:15:02,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:02,763 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:02,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:02,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169fb9c0b29e494bb0b16cb592a3720720_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773701071/Put/seqid=0 2024-11-16T16:15:02,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742042_1218 (size=12454) 2024-11-16T16:15:02,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:03,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:03,132 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:03,133 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/8da82ddff445415dacce3f1b1838d1ab is 175, key is test_row_0/A:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:03,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:03,190 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169fb9c0b29e494bb0b16cb592a3720720_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fb9c0b29e494bb0b16cb592a3720720_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:03,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742043_1219 (size=31869) 2024-11-16T16:15:03,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6846426076cb4683804150825bedb55b, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:03,192 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/d5956904f633480999234a5e89bcfde3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d5956904f633480999234a5e89bcfde3 2024-11-16T16:15:03,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6846426076cb4683804150825bedb55b is 175, key is test_row_0/A:col10/1731773701071/Put/seqid=0 2024-11-16T16:15:03,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742044_1220 (size=31255) 2024-11-16T16:15:03,198 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=345, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6846426076cb4683804150825bedb55b 2024-11-16T16:15:03,199 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/8da82ddff445415dacce3f1b1838d1ab as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab 2024-11-16T16:15:03,200 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into d5956904f633480999234a5e89bcfde3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:03,200 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:03,200 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=12, startTime=1731773702695; duration=0sec 2024-11-16T16:15:03,200 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:03,200 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:15:03,200 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:03,205 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49532 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:03,205 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:15:03,205 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:03,205 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/0d6d66a26dba46059f7b11c9a3cad6d3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=48.4 K 2024-11-16T16:15:03,206 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d6d66a26dba46059f7b11c9a3cad6d3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731773696992 2024-11-16T16:15:03,206 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 25d3428dfe9145719e14ed3ac93bee43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731773697740 2024-11-16T16:15:03,206 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c4fd5b0e985947f3b0f49c8b4a1a5181, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731773698687 2024-11-16T16:15:03,207 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 46625305ada0463f95f6f367c119fe8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:03,209 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into 8da82ddff445415dacce3f1b1838d1ab(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:03,209 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:03,209 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=12, startTime=1731773702695; duration=0sec 2024-11-16T16:15:03,210 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:03,210 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:15:03,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eb79e05c6cb54915ab7c728a48550072 is 50, key is test_row_0/B:col10/1731773701071/Put/seqid=0 2024-11-16T16:15:03,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742045_1221 (size=12301) 2024-11-16T16:15:03,218 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eb79e05c6cb54915ab7c728a48550072 2024-11-16T16:15:03,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:03,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:03,230 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:03,230 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/d00782e8ec5245a99674bec946bd515f is 50, key is test_row_0/C:col10/1731773699873/Put/seqid=0 2024-11-16T16:15:03,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/76594c8667f74732acde46886bb0e235 is 50, key is test_row_0/C:col10/1731773701071/Put/seqid=0 2024-11-16T16:15:03,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742047_1223 (size=12301) 2024-11-16T16:15:03,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773763257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,261 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773763257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742046_1222 (size=12915) 2024-11-16T16:15:03,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773763365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773763365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773763574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773763574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:03,660 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/76594c8667f74732acde46886bb0e235 2024-11-16T16:15:03,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6846426076cb4683804150825bedb55b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b 2024-11-16T16:15:03,669 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b, entries=150, sequenceid=345, filesize=30.5 K 2024-11-16T16:15:03,669 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/d00782e8ec5245a99674bec946bd515f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d00782e8ec5245a99674bec946bd515f 2024-11-16T16:15:03,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eb79e05c6cb54915ab7c728a48550072 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072 2024-11-16T16:15:03,675 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into d00782e8ec5245a99674bec946bd515f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:03,676 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:03,676 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=12, startTime=1731773702695; duration=0sec 2024-11-16T16:15:03,676 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:03,676 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:15:03,677 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T16:15:03,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/76594c8667f74732acde46886bb0e235 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235 2024-11-16T16:15:03,682 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T16:15:03,683 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 3ed8b6f0ed534a06346c44e18fa12f8f in 921ms, sequenceid=345, compaction requested=false 2024-11-16T16:15:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-16T16:15:03,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-16T16:15:03,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-16T16:15:03,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-16T16:15:03,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.2300 sec 2024-11-16T16:15:03,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:03,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T16:15:03,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:03,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:03,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162d529d4a18cb4807a6531779c56ede30_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:03,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742048_1224 (size=12454) 2024-11-16T16:15:03,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773763916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:03,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:03,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773763917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:03,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773764025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773764025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773764235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773764235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,299 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:04,303 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162d529d4a18cb4807a6531779c56ede30_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162d529d4a18cb4807a6531779c56ede30_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:04,304 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/2a72a267bcf047c1866a6f07b9d73588, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:04,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/2a72a267bcf047c1866a6f07b9d73588 is 175, key is test_row_0/A:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742049_1225 (size=31255) 2024-11-16T16:15:04,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773764545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773764545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:04,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-16T16:15:04,605 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:04,606 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-16T16:15:04,607 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:04,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:04,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:04,708 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=363, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/2a72a267bcf047c1866a6f07b9d73588 2024-11-16T16:15:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:04,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/275f9e61976748cebb388e8c264dcdb2 is 50, key is test_row_0/B:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:04,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742050_1226 (size=12301) 2024-11-16T16:15:04,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:04,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:04,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773765055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:05,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773765055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:05,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/275f9e61976748cebb388e8c264dcdb2 2024-11-16T16:15:05,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/946974c190d54ee6a0fb5990625f1cfe is 50, key is test_row_0/C:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:05,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742051_1227 (size=12301) 2024-11-16T16:15:05,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:05,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:05,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:05,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:05,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/946974c190d54ee6a0fb5990625f1cfe 2024-11-16T16:15:05,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/2a72a267bcf047c1866a6f07b9d73588 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588 2024-11-16T16:15:05,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588, entries=150, sequenceid=363, filesize=30.5 K 2024-11-16T16:15:05,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/275f9e61976748cebb388e8c264dcdb2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2 2024-11-16T16:15:05,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2, entries=150, sequenceid=363, filesize=12.0 K 2024-11-16T16:15:05,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/946974c190d54ee6a0fb5990625f1cfe as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe 2024-11-16T16:15:05,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe, entries=150, sequenceid=363, filesize=12.0 K 2024-11-16T16:15:05,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1660ms, sequenceid=363, compaction requested=true 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:05,548 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:05,548 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:05,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:05,549 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:05,549 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:05,549 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:15:05,549 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:15:05,549 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,549 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,549 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=92.2 K 2024-11-16T16:15:05,549 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d5956904f633480999234a5e89bcfde3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.6 K 2024-11-16T16:15:05,549 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,549 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588] 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8da82ddff445415dacce3f1b1838d1ab, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d5956904f633480999234a5e89bcfde3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting eb79e05c6cb54915ab7c728a48550072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773701051 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6846426076cb4683804150825bedb55b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773701051 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 275f9e61976748cebb388e8c264dcdb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:05,550 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a72a267bcf047c1866a6f07b9d73588, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:05,555 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:05,556 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#191 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:05,557 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c122ba39eb914caa882768368378ab34 is 50, key is test_row_0/B:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:05,558 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116732e52f3864449b4acdcb5dd558fbd37_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:05,560 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116732e52f3864449b4acdcb5dd558fbd37_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:05,560 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116732e52f3864449b4acdcb5dd558fbd37_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:05,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742052_1228 (size=13017) 2024-11-16T16:15:05,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742053_1229 (size=4469) 2024-11-16T16:15:05,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,677 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:05,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:05,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111685eff8536f9d4b3a9cb7b80a5ebf70b6_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773703916/Put/seqid=0 2024-11-16T16:15:05,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742054_1230 (size=12454) 2024-11-16T16:15:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:05,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. as already flushing 2024-11-16T16:15:05,976 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/c122ba39eb914caa882768368378ab34 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c122ba39eb914caa882768368378ab34 2024-11-16T16:15:05,977 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#192 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:05,978 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/ac5f1482a45e4c3f8922b28b732cf0ec is 175, key is test_row_0/A:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:05,982 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into c122ba39eb914caa882768368378ab34(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:05,982 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:05,982 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=13, startTime=1731773705548; duration=0sec 2024-11-16T16:15:05,982 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:05,982 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:15:05,982 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:05,983 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:05,983 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:15:05,984 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:05,984 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d00782e8ec5245a99674bec946bd515f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.6 K 2024-11-16T16:15:05,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742055_1231 (size=31971) 2024-11-16T16:15:05,984 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d00782e8ec5245a99674bec946bd515f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1731773699873 2024-11-16T16:15:05,984 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 76594c8667f74732acde46886bb0e235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773701051 2024-11-16T16:15:05,985 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 946974c190d54ee6a0fb5990625f1cfe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:05,992 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:05,993 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/31d7550f07ad4db8966454d8fe12827a is 50, key is test_row_0/C:col10/1731773703255/Put/seqid=0 2024-11-16T16:15:05,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773765990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:05,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:05,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742056_1232 (size=13017) 2024-11-16T16:15:06,003 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/31d7550f07ad4db8966454d8fe12827a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/31d7550f07ad4db8966454d8fe12827a 2024-11-16T16:15:06,009 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into 31d7550f07ad4db8966454d8fe12827a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:06,009 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:06,009 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=13, startTime=1731773705548; duration=0sec 2024-11-16T16:15:06,010 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:06,010 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:15:06,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57496 deadline: 1731773766064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:06,065 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:06,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57448 deadline: 1731773766075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:06,076 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3ed8b6f0ed534a06346c44e18fa12f8f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:06,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:06,091 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111685eff8536f9d4b3a9cb7b80a5ebf70b6_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685eff8536f9d4b3a9cb7b80a5ebf70b6_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:06,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/d9dc22c977d14877ade1ffd4e9f30ef2, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:06,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/d9dc22c977d14877ade1ffd4e9f30ef2 is 175, key is test_row_0/A:col10/1731773703916/Put/seqid=0 2024-11-16T16:15:06,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742057_1233 (size=31255) 2024-11-16T16:15:06,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773766105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:06,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773766315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:06,388 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/ac5f1482a45e4c3f8922b28b732cf0ec as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec 2024-11-16T16:15:06,393 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into ac5f1482a45e4c3f8922b28b732cf0ec(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:06,393 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:06,393 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=13, startTime=1731773705548; duration=0sec 2024-11-16T16:15:06,393 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:06,393 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:15:06,496 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/d9dc22c977d14877ade1ffd4e9f30ef2 2024-11-16T16:15:06,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7d702b6f42714db2aa6cbe74e05d7075 is 50, key is test_row_0/B:col10/1731773703916/Put/seqid=0 2024-11-16T16:15:06,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742058_1234 (size=12301) 2024-11-16T16:15:06,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:06,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773766625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:06,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:06,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:06,906 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7d702b6f42714db2aa6cbe74e05d7075 2024-11-16T16:15:06,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/caab1069f11a4e1aabae775bff88fc26 is 50, key is test_row_0/C:col10/1731773703916/Put/seqid=0 2024-11-16T16:15:06,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742059_1235 (size=12301) 2024-11-16T16:15:06,927 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/caab1069f11a4e1aabae775bff88fc26 2024-11-16T16:15:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/d9dc22c977d14877ade1ffd4e9f30ef2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2 2024-11-16T16:15:06,935 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2, entries=150, sequenceid=380, filesize=30.5 K 2024-11-16T16:15:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7d702b6f42714db2aa6cbe74e05d7075 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075 2024-11-16T16:15:06,940 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075, entries=150, sequenceid=380, filesize=12.0 K 2024-11-16T16:15:06,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/caab1069f11a4e1aabae775bff88fc26 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26 2024-11-16T16:15:06,946 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26, entries=150, sequenceid=380, filesize=12.0 K 2024-11-16T16:15:06,947 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1270ms, sequenceid=380, compaction requested=false 2024-11-16T16:15:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-16T16:15:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-16T16:15:06,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-16T16:15:06,950 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-16T16:15:06,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.3440 sec 2024-11-16T16:15:07,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:07,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:07,136 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166c2debed01564ca4a2c08e2f0beac833_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742060_1236 (size=12454) 2024-11-16T16:15:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:07,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773767183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:07,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:07,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773767295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:07,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57520 deadline: 1731773767505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:07,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:07,548 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:07,551 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411166c2debed01564ca4a2c08e2f0beac833_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166c2debed01564ca4a2c08e2f0beac833_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:07,553 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/9106690caa9146e4bf3d4ac257a7e166, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:07,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/9106690caa9146e4bf3d4ac257a7e166 is 175, key is test_row_0/A:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742061_1237 (size=31255) 2024-11-16T16:15:07,570 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/9106690caa9146e4bf3d4ac257a7e166 2024-11-16T16:15:07,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2fb22b1b3ec7442bae0720a1e6b3f426 is 50, key is test_row_0/B:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742062_1238 (size=12301) 2024-11-16T16:15:07,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2fb22b1b3ec7442bae0720a1e6b3f426 2024-11-16T16:15:07,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c498c745bbbf44b2b4f614e38177f505 is 50, key is test_row_0/C:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742063_1239 (size=12301) 2024-11-16T16:15:07,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c498c745bbbf44b2b4f614e38177f505 2024-11-16T16:15:07,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/9106690caa9146e4bf3d4ac257a7e166 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166 2024-11-16T16:15:07,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166, entries=150, sequenceid=399, filesize=30.5 K 2024-11-16T16:15:07,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/2fb22b1b3ec7442bae0720a1e6b3f426 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426 2024-11-16T16:15:07,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T16:15:07,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/c498c745bbbf44b2b4f614e38177f505 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505 2024-11-16T16:15:07,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505, entries=150, sequenceid=399, filesize=12.0 K 2024-11-16T16:15:07,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 3ed8b6f0ed534a06346c44e18fa12f8f in 511ms, sequenceid=399, compaction requested=true 2024-11-16T16:15:07,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:07,647 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:07,647 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3ed8b6f0ed534a06346c44e18fa12f8f:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:07,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/A is initiating minor compaction (all files) 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/B is initiating minor compaction (all files) 2024-11-16T16:15:07,648 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/A in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:07,648 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/B in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:07,648 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=92.3 K 2024-11-16T16:15:07,648 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c122ba39eb914caa882768368378ab34, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.7 K 2024-11-16T16:15:07,648 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166] 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c122ba39eb914caa882768368378ab34, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:07,648 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac5f1482a45e4c3f8922b28b732cf0ec, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:07,649 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9dc22c977d14877ade1ffd4e9f30ef2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731773703905 2024-11-16T16:15:07,649 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d702b6f42714db2aa6cbe74e05d7075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731773703905 2024-11-16T16:15:07,649 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9106690caa9146e4bf3d4ac257a7e166, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731773705978 2024-11-16T16:15:07,649 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fb22b1b3ec7442bae0720a1e6b3f426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731773705978 2024-11-16T16:15:07,656 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:07,657 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#B#compaction#200 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:07,658 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/bf472cbe22254986a9cc47877a35b3cb is 50, key is test_row_0/B:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,659 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411168cc42e00558f4f52b45e605c4044ebf1_3ed8b6f0ed534a06346c44e18fa12f8f store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:07,661 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411168cc42e00558f4f52b45e605c4044ebf1_3ed8b6f0ed534a06346c44e18fa12f8f, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:07,661 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168cc42e00558f4f52b45e605c4044ebf1_3ed8b6f0ed534a06346c44e18fa12f8f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:07,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742064_1240 (size=13119) 2024-11-16T16:15:07,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742065_1241 (size=4469) 2024-11-16T16:15:07,675 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#A#compaction#201 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:07,675 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6678fceba1db4523819feafc66a4c06c is 175, key is test_row_0/A:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:07,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742066_1242 (size=32073) 2024-11-16T16:15:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:07,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:07,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:07,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116895d689f227e4b7b92f3c6b591629106_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773707172/Put/seqid=0 2024-11-16T16:15:07,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742067_1243 (size=12454) 2024-11-16T16:15:07,834 INFO [Thread-718 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-718. 2024-11-16T16:15:07,834 DEBUG [Thread-718 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:07,834 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,834 DEBUG [Thread-718 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,834 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:07,835 INFO [Thread-714 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-714. 2024-11-16T16:15:07,835 DEBUG [Thread-714 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:07,835 DEBUG [Thread-714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 DEBUG [Thread-714 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:07,835 INFO [Thread-705 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-705. 2024-11-16T16:15:07,835 DEBUG [Thread-705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:07,835 INFO [Thread-716 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-716. 2024-11-16T16:15:07,835 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 DEBUG [Thread-716 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:07,835 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 DEBUG [Thread-705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:07,835 DEBUG [Thread-716 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,835 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:07,841 INFO [Thread-720 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-720. 2024-11-16T16:15:07,841 DEBUG [Thread-720 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:07,841 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,841 DEBUG [Thread-720 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:07,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,085 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/bf472cbe22254986a9cc47877a35b3cb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/bf472cbe22254986a9cc47877a35b3cb 2024-11-16T16:15:08,088 INFO [Thread-707 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-707. 2024-11-16T16:15:08,088 INFO [Thread-709 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-709. 2024-11-16T16:15:08,088 DEBUG [Thread-707 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:08,088 DEBUG [Thread-709 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:08,088 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,088 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,088 DEBUG [Thread-707 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,088 DEBUG [Thread-709 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,089 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/6678fceba1db4523819feafc66a4c06c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6678fceba1db4523819feafc66a4c06c 2024-11-16T16:15:08,092 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/B of 3ed8b6f0ed534a06346c44e18fa12f8f into bf472cbe22254986a9cc47877a35b3cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:08,092 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:08,092 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/B, priority=13, startTime=1731773707647; duration=0sec 2024-11-16T16:15:08,092 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:08,092 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:B 2024-11-16T16:15:08,092 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:08,093 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:08,093 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3ed8b6f0ed534a06346c44e18fa12f8f/C is initiating minor compaction (all files) 2024-11-16T16:15:08,093 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3ed8b6f0ed534a06346c44e18fa12f8f/C in TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:08,093 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/31d7550f07ad4db8966454d8fe12827a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp, totalSize=36.7 K 2024-11-16T16:15:08,093 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d7550f07ad4db8966454d8fe12827a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731773703240 2024-11-16T16:15:08,094 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting caab1069f11a4e1aabae775bff88fc26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731773703905 2024-11-16T16:15:08,094 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c498c745bbbf44b2b4f614e38177f505, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731773705978 2024-11-16T16:15:08,094 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/A of 3ed8b6f0ed534a06346c44e18fa12f8f into 6678fceba1db4523819feafc66a4c06c(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:08,094 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:08,094 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/A, priority=13, startTime=1731773707646; duration=0sec 2024-11-16T16:15:08,094 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:08,094 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:A 2024-11-16T16:15:08,100 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3ed8b6f0ed534a06346c44e18fa12f8f#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:08,101 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/7a0616ba450840f684a9603c189ac6c4 is 50, key is test_row_0/C:col10/1731773705985/Put/seqid=0 2024-11-16T16:15:08,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742068_1244 (size=13119) 2024-11-16T16:15:08,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:08,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116895d689f227e4b7b92f3c6b591629106_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116895d689f227e4b7b92f3c6b591629106_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:08,288 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/12048fd6622941e49cc5886381243afd, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:08,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/12048fd6622941e49cc5886381243afd is 175, key is test_row_0/A:col10/1731773707172/Put/seqid=0 2024-11-16T16:15:08,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742069_1245 (size=31255) 2024-11-16T16:15:08,520 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/7a0616ba450840f684a9603c189ac6c4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/7a0616ba450840f684a9603c189ac6c4 2024-11-16T16:15:08,526 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3ed8b6f0ed534a06346c44e18fa12f8f/C of 3ed8b6f0ed534a06346c44e18fa12f8f into 7a0616ba450840f684a9603c189ac6c4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:08,526 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:08,526 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., storeName=3ed8b6f0ed534a06346c44e18fa12f8f/C, priority=13, startTime=1731773707647; duration=0sec 2024-11-16T16:15:08,526 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:08,527 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3ed8b6f0ed534a06346c44e18fa12f8f:C 2024-11-16T16:15:08,527 INFO [Thread-703 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-703. 2024-11-16T16:15:08,527 DEBUG [Thread-703 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:08,527 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,527 DEBUG [Thread-703 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,558 INFO [Thread-711 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-711. 2024-11-16T16:15:08,558 DEBUG [Thread-711 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:08,558 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,558 DEBUG [Thread-711 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,697 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/12048fd6622941e49cc5886381243afd 2024-11-16T16:15:08,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eed18a8c1627439493899969fc46b382 is 50, key is test_row_0/B:col10/1731773707172/Put/seqid=0 2024-11-16T16:15:08,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742070_1246 (size=12301) 2024-11-16T16:15:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-16T16:15:08,766 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:08,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:15:08,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-16T16:15:08,766 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3064 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2976 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1292 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3876 rows 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1300 2024-11-16T16:15:08,767 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3900 rows 2024-11-16T16:15:08,767 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:15:08,768 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:15:08,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,768 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:08,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:08,769 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:15:08,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:08,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:08,772 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773708771"}]},"ts":"1731773708771"} 2024-11-16T16:15:08,773 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:15:08,773 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:15:08,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:15:08,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, UNASSIGN}] 2024-11-16T16:15:08,775 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, UNASSIGN 2024-11-16T16:15:08,776 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:08,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, UNASSIGN because future has completed 2024-11-16T16:15:08,777 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:15:08,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:08,930 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:08,930 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:15:08,931 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 3ed8b6f0ed534a06346c44e18fa12f8f, disabling compactions & flushes 2024-11-16T16:15:08,931 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:09,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eed18a8c1627439493899969fc46b382 2024-11-16T16:15:09,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/591bd33dc1ed4d3d9ccef46d91b13af2 is 50, key is test_row_0/C:col10/1731773707172/Put/seqid=0 2024-11-16T16:15:09,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742071_1247 (size=12301) 2024-11-16T16:15:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:09,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/591bd33dc1ed4d3d9ccef46d91b13af2 2024-11-16T16:15:09,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/12048fd6622941e49cc5886381243afd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/12048fd6622941e49cc5886381243afd 2024-11-16T16:15:09,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/12048fd6622941e49cc5886381243afd, entries=150, sequenceid=416, filesize=30.5 K 2024-11-16T16:15:09,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/eed18a8c1627439493899969fc46b382 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eed18a8c1627439493899969fc46b382 2024-11-16T16:15:09,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eed18a8c1627439493899969fc46b382, entries=150, sequenceid=416, filesize=12.0 K 2024-11-16T16:15:09,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/591bd33dc1ed4d3d9ccef46d91b13af2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/591bd33dc1ed4d3d9ccef46d91b13af2 2024-11-16T16:15:09,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/591bd33dc1ed4d3d9ccef46d91b13af2, entries=150, sequenceid=416, filesize=12.0 K 2024-11-16T16:15:09,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=60.38 KB/61830 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1742ms, sequenceid=416, compaction requested=false 2024-11-16T16:15:09,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3ed8b6f0ed534a06346c44e18fa12f8f: 2024-11-16T16:15:09,561 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. after waiting 0 ms 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:09,561 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 3ed8b6f0ed534a06346c44e18fa12f8f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=A 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=B 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3ed8b6f0ed534a06346c44e18fa12f8f, store=C 2024-11-16T16:15:09,561 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:09,567 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116aefe9ec149e047f8809c6d5974bd8c97_3ed8b6f0ed534a06346c44e18fa12f8f is 50, key is test_row_0/A:col10/1731773708085/Put/seqid=0 2024-11-16T16:15:09,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742072_1248 (size=12454) 2024-11-16T16:15:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:09,972 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:09,982 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116aefe9ec149e047f8809c6d5974bd8c97_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aefe9ec149e047f8809c6d5974bd8c97_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:09,983 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/883dd3fdee9b4739aa0a1c57d2abb5ed, store: [table=TestAcidGuarantees family=A region=3ed8b6f0ed534a06346c44e18fa12f8f] 2024-11-16T16:15:09,984 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/883dd3fdee9b4739aa0a1c57d2abb5ed is 175, key is test_row_0/A:col10/1731773708085/Put/seqid=0 2024-11-16T16:15:09,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742073_1249 (size=31255) 2024-11-16T16:15:10,390 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=431, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/883dd3fdee9b4739aa0a1c57d2abb5ed 2024-11-16T16:15:10,403 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7b54c686c92648fcb5ea63859d9e7e88 is 50, key is test_row_0/B:col10/1731773708085/Put/seqid=0 2024-11-16T16:15:10,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742074_1250 (size=12301) 2024-11-16T16:15:10,810 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7b54c686c92648fcb5ea63859d9e7e88 2024-11-16T16:15:10,823 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/dc0fae1f776e4e009dc966b402bb5125 is 50, key is test_row_0/C:col10/1731773708085/Put/seqid=0 2024-11-16T16:15:10,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742075_1251 (size=12301) 2024-11-16T16:15:10,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:15:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:11,230 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/dc0fae1f776e4e009dc966b402bb5125 2024-11-16T16:15:11,238 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/A/883dd3fdee9b4739aa0a1c57d2abb5ed as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/883dd3fdee9b4739aa0a1c57d2abb5ed 2024-11-16T16:15:11,244 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/883dd3fdee9b4739aa0a1c57d2abb5ed, entries=150, sequenceid=431, filesize=30.5 K 2024-11-16T16:15:11,245 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/B/7b54c686c92648fcb5ea63859d9e7e88 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7b54c686c92648fcb5ea63859d9e7e88 2024-11-16T16:15:11,249 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7b54c686c92648fcb5ea63859d9e7e88, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T16:15:11,250 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/.tmp/C/dc0fae1f776e4e009dc966b402bb5125 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/dc0fae1f776e4e009dc966b402bb5125 2024-11-16T16:15:11,254 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/dc0fae1f776e4e009dc966b402bb5125, entries=150, sequenceid=431, filesize=12.0 K 2024-11-16T16:15:11,255 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1694ms, sequenceid=431, compaction requested=true 2024-11-16T16:15:11,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166] to archive 2024-11-16T16:15:11,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:11,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/338a83c138704b08b47eced06b25e419 2024-11-16T16:15:11,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/4d994c64c4e64166b0ad9e1dfcb75be6 2024-11-16T16:15:11,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/09b8e97302fe4753b6deee8062d1b276 2024-11-16T16:15:11,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/3ac188e6edef48bf92c7da6af1303572 2024-11-16T16:15:11,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/76a95fcf14a045ce8c644ed647ef7695 2024-11-16T16:15:11,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/932f1bfce414433588c940332c8b329f 2024-11-16T16:15:11,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0a8593b94240490dbab88249898e7740 2024-11-16T16:15:11,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/0268f04d24ea4635a09d554a619c83d8 2024-11-16T16:15:11,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ba388f9fba1a4d85a0189f6d2f6a9d93 2024-11-16T16:15:11,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/f0f7380f77fe46ea9e4a6ef88aa3f4fb 2024-11-16T16:15:11,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6536206cccb54b538b18c86542c91e7a 2024-11-16T16:15:11,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/b6f597b9095849169bcf02f38e1feda8 2024-11-16T16:15:11,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/38bf796f0bc64b3ba53b99233e8335ff 2024-11-16T16:15:11,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/1565c5f8c8504eba81de4ef0266ecb96 2024-11-16T16:15:11,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/78b1266ab7ea452682523d34c8eb9287 2024-11-16T16:15:11,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/88be1bb998ab4d26b6bfba7dba8a2ed5 2024-11-16T16:15:11,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/450299222db44e2cb45911ac07c81579 2024-11-16T16:15:11,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/44989c2f88c04f829f00e3deebc371d4 2024-11-16T16:15:11,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/8da82ddff445415dacce3f1b1838d1ab 2024-11-16T16:15:11,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6846426076cb4683804150825bedb55b 2024-11-16T16:15:11,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/ac5f1482a45e4c3f8922b28b732cf0ec 2024-11-16T16:15:11,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/2a72a267bcf047c1866a6f07b9d73588 2024-11-16T16:15:11,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/d9dc22c977d14877ade1ffd4e9f30ef2 2024-11-16T16:15:11,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/9106690caa9146e4bf3d4ac257a7e166 2024-11-16T16:15:11,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/a34868831eaa48caa998206b39b09e72, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/84d2a149df6a4233bbe24988f52b2839, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/af2114703ad0481bb9841e702f5ca923, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/274fd944e89b4f4f9383591b2a1fe2c4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d5956904f633480999234a5e89bcfde3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c122ba39eb914caa882768368378ab34, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426] to archive 2024-11-16T16:15:11,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:11,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c5f66c0f4db24ac785a6fd25d42809f5 2024-11-16T16:15:11,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/337001c849cb4b4fa68e1de90430fe62 2024-11-16T16:15:11,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2d2438b76592493194dd5c24f9ad5dfd 2024-11-16T16:15:11,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/a34868831eaa48caa998206b39b09e72 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/a34868831eaa48caa998206b39b09e72 2024-11-16T16:15:11,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/34abf37793aa4565b64d149413282fc6 2024-11-16T16:15:11,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/556dead5271741878af0799901170739 2024-11-16T16:15:11,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c0c9ab9b448649908b0563b969f8d066 2024-11-16T16:15:11,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/84d2a149df6a4233bbe24988f52b2839 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/84d2a149df6a4233bbe24988f52b2839 2024-11-16T16:15:11,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7f7954f45a0a40529641d95c507a32b4 2024-11-16T16:15:11,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/01fadb0783b34305a774ab97f8207a59 2024-11-16T16:15:11,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/af2114703ad0481bb9841e702f5ca923 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/af2114703ad0481bb9841e702f5ca923 2024-11-16T16:15:11,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/acefdcdf29eb43bc8fd9b1541d00478e 2024-11-16T16:15:11,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d9fd9acb387741ae82e5b4910c6339fd 2024-11-16T16:15:11,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/274fd944e89b4f4f9383591b2a1fe2c4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/274fd944e89b4f4f9383591b2a1fe2c4 2024-11-16T16:15:11,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/1419f879214d4826b13d5dd65fa58bf5 2024-11-16T16:15:11,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6f48aa997b62459984aa45c9d1f364b2 2024-11-16T16:15:11,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c9e0e18988a24b6f815db782d0cc8193 2024-11-16T16:15:11,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d5956904f633480999234a5e89bcfde3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/d5956904f633480999234a5e89bcfde3 2024-11-16T16:15:11,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/6210b1552ab546bc8995de9ef52f6223 2024-11-16T16:15:11,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eb79e05c6cb54915ab7c728a48550072 2024-11-16T16:15:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c122ba39eb914caa882768368378ab34 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/c122ba39eb914caa882768368378ab34 2024-11-16T16:15:11,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/275f9e61976748cebb388e8c264dcdb2 2024-11-16T16:15:11,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7d702b6f42714db2aa6cbe74e05d7075 2024-11-16T16:15:11,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/2fb22b1b3ec7442bae0720a1e6b3f426 2024-11-16T16:15:11,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/1893c09b91a04f09834e7ece5e43f7df, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d11dbbbc4e9b4096b4ccbdc2d292127d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/479093064c0545ab8fc57e911e3d490e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/0d6d66a26dba46059f7b11c9a3cad6d3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d00782e8ec5245a99674bec946bd515f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/31d7550f07ad4db8966454d8fe12827a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505] to archive 2024-11-16T16:15:11,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:11,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/8f59d56c7bc040eebefdf73ec63f2864 2024-11-16T16:15:11,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/fba367d81af14e948ccd4b18a58ae802 2024-11-16T16:15:11,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5c75fb608726440087e339d3f8ffcd39 2024-11-16T16:15:11,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/1893c09b91a04f09834e7ece5e43f7df to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/1893c09b91a04f09834e7ece5e43f7df 2024-11-16T16:15:11,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/5fca36ab18df4a08aa72203c62da869b 2024-11-16T16:15:11,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/039c745877984048b63f70cd03227c9b 2024-11-16T16:15:11,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/55029b05bc924f40b3fd0a8adf3b01ab 2024-11-16T16:15:11,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d11dbbbc4e9b4096b4ccbdc2d292127d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d11dbbbc4e9b4096b4ccbdc2d292127d 2024-11-16T16:15:11,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/de424e2670d14a2584749e0158e3fb6b 2024-11-16T16:15:11,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/2fae6cfd3b2343cdbb0bcd5c2c86f48d 2024-11-16T16:15:11,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/479093064c0545ab8fc57e911e3d490e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/479093064c0545ab8fc57e911e3d490e 2024-11-16T16:15:11,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/f4267b0909594f3d897146cfc08eef71 2024-11-16T16:15:11,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/aa3b5457289b43929684f812255ce3fa 2024-11-16T16:15:11,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/0d6d66a26dba46059f7b11c9a3cad6d3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/0d6d66a26dba46059f7b11c9a3cad6d3 2024-11-16T16:15:11,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/83e1619c476d40e18a299235c7fcff03 2024-11-16T16:15:11,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/25d3428dfe9145719e14ed3ac93bee43 2024-11-16T16:15:11,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c4fd5b0e985947f3b0f49c8b4a1a5181 2024-11-16T16:15:11,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d00782e8ec5245a99674bec946bd515f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/d00782e8ec5245a99674bec946bd515f 2024-11-16T16:15:11,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/46625305ada0463f95f6f367c119fe8f 2024-11-16T16:15:11,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/76594c8667f74732acde46886bb0e235 2024-11-16T16:15:11,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/31d7550f07ad4db8966454d8fe12827a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/31d7550f07ad4db8966454d8fe12827a 2024-11-16T16:15:11,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/946974c190d54ee6a0fb5990625f1cfe 2024-11-16T16:15:11,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/caab1069f11a4e1aabae775bff88fc26 2024-11-16T16:15:11,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/c498c745bbbf44b2b4f614e38177f505 2024-11-16T16:15:11,341 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits/434.seqid, newMaxSeqId=434, maxSeqId=4 2024-11-16T16:15:11,342 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. 2024-11-16T16:15:11,342 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 3ed8b6f0ed534a06346c44e18fa12f8f: Waiting for close lock at 1731773708931Running coprocessor pre-close hooks at 1731773708931Disabling compacts and flushes for region at 1731773708931Disabling writes for close at 1731773709561 (+630 ms)Obtaining lock to block concurrent updates at 1731773709561Preparing flush snapshotting stores in 3ed8b6f0ed534a06346c44e18fa12f8f at 1731773709561Finished memstore snapshotting TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f., syncing WAL and waiting on mvcc, flushsize=dataSize=61830, getHeapSize=184320, getOffHeapSize=0, getCellsCount=1350 at 1731773709561Flushing stores of TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f. at 1731773709562 (+1 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/A: creating writer at 1731773709562Flushing mob file 3ed8b6f0ed534a06346c44e18fa12f8f/A: appending metadata at 1731773709567 (+5 ms)Flushing mob file 3ed8b6f0ed534a06346c44e18fa12f8f/A: closing flushed file at 1731773709567Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/A: appending metadata at 1731773709983 (+416 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/A: closing flushed file at 1731773709983Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/B: creating writer at 1731773710396 (+413 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/B: appending metadata at 1731773710402 (+6 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/B: closing flushed file at 1731773710402Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/C: creating writer at 1731773710819 (+417 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/C: appending metadata at 1731773710823 (+4 ms)Flushing 3ed8b6f0ed534a06346c44e18fa12f8f/C: closing flushed file at 1731773710823Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26bdf9ea: reopening flushed file at 1731773711237 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c90b121: reopening flushed file at 1731773711244 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39b08c86: reopening flushed file at 1731773711250 (+6 ms)Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 3ed8b6f0ed534a06346c44e18fa12f8f in 1694ms, sequenceid=431, compaction requested=true at 1731773711255 (+5 ms)Writing region close event to WAL at 1731773711338 (+83 ms)Running coprocessor post-close hooks at 1731773711341 (+3 ms)Closed at 1731773711342 (+1 ms) 2024-11-16T16:15:11,344 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:11,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=3ed8b6f0ed534a06346c44e18fa12f8f, regionState=CLOSED 2024-11-16T16:15:11,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:11,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-16T16:15:11,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 3ed8b6f0ed534a06346c44e18fa12f8f, server=303fe01965e8,37355,1731773652566 in 2.5690 sec 2024-11-16T16:15:11,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-16T16:15:11,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3ed8b6f0ed534a06346c44e18fa12f8f, UNASSIGN in 2.5730 sec 2024-11-16T16:15:11,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-16T16:15:11,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5750 sec 2024-11-16T16:15:11,350 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773711350"}]},"ts":"1731773711350"} 2024-11-16T16:15:11,351 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:15:11,351 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:15:11,353 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5830 sec 2024-11-16T16:15:12,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:15:12,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-16T16:15:12,936 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:12,937 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:15:12,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:12,942 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:12,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T16:15:12,944 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:12,948 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,952 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits] 2024-11-16T16:15:12,955 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/12048fd6622941e49cc5886381243afd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/12048fd6622941e49cc5886381243afd 2024-11-16T16:15:12,957 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6678fceba1db4523819feafc66a4c06c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/6678fceba1db4523819feafc66a4c06c 2024-11-16T16:15:12,959 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/883dd3fdee9b4739aa0a1c57d2abb5ed to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/A/883dd3fdee9b4739aa0a1c57d2abb5ed 2024-11-16T16:15:12,961 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7b54c686c92648fcb5ea63859d9e7e88 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/7b54c686c92648fcb5ea63859d9e7e88 2024-11-16T16:15:12,963 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/bf472cbe22254986a9cc47877a35b3cb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/bf472cbe22254986a9cc47877a35b3cb 2024-11-16T16:15:12,964 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eed18a8c1627439493899969fc46b382 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/B/eed18a8c1627439493899969fc46b382 2024-11-16T16:15:12,966 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/591bd33dc1ed4d3d9ccef46d91b13af2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/591bd33dc1ed4d3d9ccef46d91b13af2 2024-11-16T16:15:12,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/7a0616ba450840f684a9603c189ac6c4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/7a0616ba450840f684a9603c189ac6c4 2024-11-16T16:15:12,967 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/dc0fae1f776e4e009dc966b402bb5125 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/C/dc0fae1f776e4e009dc966b402bb5125 2024-11-16T16:15:12,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits/434.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f/recovered.edits/434.seqid 2024-11-16T16:15:12,970 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,970 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:15:12,971 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:15:12,971 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T16:15:12,975 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623002782d2a84177952df55f7376cd81_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111623002782d2a84177952df55f7376cd81_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,976 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162d529d4a18cb4807a6531779c56ede30_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411162d529d4a18cb4807a6531779c56ede30_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,977 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636a721392bc64565b11a4131276e1e94_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111636a721392bc64565b11a4131276e1e94_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,978 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116455bf6d7020b4f369187b49ad1d62607_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116455bf6d7020b4f369187b49ad1d62607_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,979 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ec3da5c62564489bff6ac3223a5a241_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164ec3da5c62564489bff6ac3223a5a241_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,980 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166c2debed01564ca4a2c08e2f0beac833_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166c2debed01564ca4a2c08e2f0beac833_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,981 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166ca01d243d5f4e2284fc679718320710_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411166ca01d243d5f4e2284fc679718320710_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,982 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8024960856489ea809cb88ebe0a912_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167a8024960856489ea809cb88ebe0a912_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,983 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685a88180c4174681b5489e622268a31b_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685a88180c4174681b5489e622268a31b_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,984 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685eff8536f9d4b3a9cb7b80a5ebf70b6_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111685eff8536f9d4b3a9cb7b80a5ebf70b6_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,985 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116895d689f227e4b7b92f3c6b591629106_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116895d689f227e4b7b92f3c6b591629106_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,986 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fb9c0b29e494bb0b16cb592a3720720_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169fb9c0b29e494bb0b16cb592a3720720_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,987 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a38d0e5bda3243b7881f9f7a2a13184c_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116a38d0e5bda3243b7881f9f7a2a13184c_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,988 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aefe9ec149e047f8809c6d5974bd8c97_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116aefe9ec149e047f8809c6d5974bd8c97_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,989 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c28257a502bf4653b52d331c90cceccb_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c28257a502bf4653b52d331c90cceccb_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,990 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbc39bc766d0480080f37ec8fa584e7f_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dbc39bc766d0480080f37ec8fa584e7f_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,991 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ddf8a6ae1488420a803f6ed2c851ef79_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ddf8a6ae1488420a803f6ed2c851ef79_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,992 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4ccaf2dd46f451a9fcb49e953f84db7_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4ccaf2dd46f451a9fcb49e953f84db7_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,993 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e559c106a5e84e64abc4dd3105f01959_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e559c106a5e84e64abc4dd3105f01959_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,994 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd208f0edc0411f9e448a5011e7707b_3ed8b6f0ed534a06346c44e18fa12f8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116efd208f0edc0411f9e448a5011e7707b_3ed8b6f0ed534a06346c44e18fa12f8f 2024-11-16T16:15:12,994 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:15:12,996 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:12,998 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:15:13,000 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:15:13,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:13,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:15:13,001 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773713001"}]},"ts":"9223372036854775807"} 2024-11-16T16:15:13,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:15:13,003 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3ed8b6f0ed534a06346c44e18fa12f8f, NAME => 'TestAcidGuarantees,,1731773684641.3ed8b6f0ed534a06346c44e18fa12f8f.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:15:13,003 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:15:13,003 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773713003"}]},"ts":"9223372036854775807"} 2024-11-16T16:15:13,004 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:15:13,005 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:13,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-16T16:15:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-16T16:15:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:15:13,055 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:13,065 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250 (was 249) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1459812294_22 at /127.0.0.1:60904 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-500171457_22 at /127.0.0.1:56578 [Waiting for operation #571] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=401 (was 365) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3876 (was 4045) 2024-11-16T16:15:13,075 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=401, ProcessCount=11, AvailableMemoryMB=3876 2024-11-16T16:15:13,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:15:13,077 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:15:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:13,079 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:15:13,079 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:13,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-16T16:15:13,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:13,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:15:13,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742076_1252 (size=1061) 2024-11-16T16:15:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:13,494 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:15:13,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742077_1253 (size=53) 2024-11-16T16:15:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:13,951 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:15:13,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1161dcb397ffa3f05e58f6711d26c4bb, disabling compactions & flushes 2024-11-16T16:15:13,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:13,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:13,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. after waiting 0 ms 2024-11-16T16:15:13,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:13,952 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:13,952 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1161dcb397ffa3f05e58f6711d26c4bb: Waiting for close lock at 1731773713951Disabling compacts and flushes for region at 1731773713951Disabling writes for close at 1731773713952 (+1 ms)Writing region close event to WAL at 1731773713952Closed at 1731773713952 2024-11-16T16:15:13,956 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:15:13,958 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773713956"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773713956"}]},"ts":"1731773713956"} 2024-11-16T16:15:13,960 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:15:13,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:15:13,961 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773713961"}]},"ts":"1731773713961"} 2024-11-16T16:15:13,962 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:15:13,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, ASSIGN}] 2024-11-16T16:15:13,964 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, ASSIGN 2024-11-16T16:15:13,965 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:15:14,116 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1161dcb397ffa3f05e58f6711d26c4bb, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:14,121 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, ASSIGN because future has completed 2024-11-16T16:15:14,121 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:14,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:14,280 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:14,281 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:15:14,281 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,281 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:15:14,282 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,282 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,284 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,285 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:14,286 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1161dcb397ffa3f05e58f6711d26c4bb columnFamilyName A 2024-11-16T16:15:14,286 DEBUG [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:14,287 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(327): Store=1161dcb397ffa3f05e58f6711d26c4bb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:14,287 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,289 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:14,289 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1161dcb397ffa3f05e58f6711d26c4bb columnFamilyName B 2024-11-16T16:15:14,289 DEBUG [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:14,290 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(327): Store=1161dcb397ffa3f05e58f6711d26c4bb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:14,290 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,291 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:14,292 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1161dcb397ffa3f05e58f6711d26c4bb columnFamilyName C 2024-11-16T16:15:14,292 DEBUG [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:14,292 INFO [StoreOpener-1161dcb397ffa3f05e58f6711d26c4bb-1 {}] regionserver.HStore(327): Store=1161dcb397ffa3f05e58f6711d26c4bb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:14,292 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:14,292 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,293 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,293 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,294 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,294 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,294 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:15:14,295 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,297 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:15:14,297 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 1161dcb397ffa3f05e58f6711d26c4bb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75269809, jitterRate=0.12160755693912506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:15:14,297 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:14,298 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 1161dcb397ffa3f05e58f6711d26c4bb: Running coprocessor pre-open hook at 1731773714282Writing region info on filesystem at 1731773714282Initializing all the Stores at 1731773714283 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773714283Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773714283Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773714284 (+1 ms)Cleaning up temporary data from old regions at 1731773714294 (+10 ms)Running coprocessor post-open hooks at 1731773714297 (+3 ms)Region opened successfully at 1731773714298 (+1 ms) 2024-11-16T16:15:14,298 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., pid=65, masterSystemTime=1731773714276 2024-11-16T16:15:14,300 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:14,300 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:14,300 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1161dcb397ffa3f05e58f6711d26c4bb, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:14,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:14,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-16T16:15:14,303 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 in 181 msec 2024-11-16T16:15:14,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-16T16:15:14,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, ASSIGN in 340 msec 2024-11-16T16:15:14,305 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:15:14,306 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773714306"}]},"ts":"1731773714306"} 2024-11-16T16:15:14,307 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:15:14,308 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:15:14,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2310 sec 2024-11-16T16:15:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-16T16:15:15,226 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:15,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,229 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d91976a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,231 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f3ba15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,233 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,234 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,238 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:15:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:15:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28a8bf55 2024-11-16T16:15:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:15:15,240 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:15:15,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9b78d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@620b0fd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,246 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1148cbeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,247 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,247 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,248 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46778175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,249 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,249 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@100eeb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,250 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,251 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d425c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,252 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514d8105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,253 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@744f723a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,254 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,255 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f51ccc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,256 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab5f52d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,257 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,257 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,257 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47161891, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6944f1b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a44a684, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f832ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac1f30b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,264 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,264 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465c3d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,266 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e954040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,267 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34476c46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,268 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:15,268 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:15,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:15,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3815287f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:15,269 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:15,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:15,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50de6c2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:15,270 DEBUG [Thread-1173 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,270 DEBUG [Thread-1182 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,270 DEBUG [Thread-1180 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,271 DEBUG [Thread-1186 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,271 DEBUG [Thread-1169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:15,271 DEBUG [Thread-1188 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,271 DEBUG [Thread-1177 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,272 DEBUG [Thread-1175 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,272 DEBUG [Thread-1184 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,272 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,273 DEBUG [Thread-1171 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:15,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,273 DEBUG [Thread-1184 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,274 DEBUG [Thread-1184 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,274 DEBUG [Thread-1177 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T16:15:15,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,274 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:15,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:15,275 DEBUG [Thread-1177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:15,276 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:15,276 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:15,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:15,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,281 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:15:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:15,287 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:15:15,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:15,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:15,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:15,287 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:15,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:15,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ae26508ba8c4479ea93e3f974f5d1bbf is 50, key is test_row_0/A:col10/1731773715281/Put/seqid=0 2024-11-16T16:15:15,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731773715299, max timestamp=1731773715315Num uniques 859; 2024-11-16T16:15:15,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731773715299, max timestamp=1731773715315Num uniques 859; 2024-11-16T16:15:15,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:15,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773715299, max timestamp=1731773715315Num uniques 859; 2024-11-16T16:15:15,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:15,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742078_1254 (size=12001) 2024-11-16T16:15:15,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773775338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773775338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,347 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773775345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773775347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773775347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T16:15:15,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T16:15:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773775455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773775455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773775455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773775456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773775456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T16:15:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T16:15:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773775664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773775664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773775664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773775665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773775665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ae26508ba8c4479ea93e3f974f5d1bbf 2024-11-16T16:15:15,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T16:15:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:15,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:15,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/9890c63c69b44cc09b653685fdf55419 is 50, key is test_row_0/B:col10/1731773715281/Put/seqid=0 2024-11-16T16:15:15,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742079_1255 (size=12001) 2024-11-16T16:15:15,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/9890c63c69b44cc09b653685fdf55419 2024-11-16T16:15:15,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f38486fa14a04227ab0683d1757477cc is 50, key is test_row_0/C:col10/1731773715281/Put/seqid=0 2024-11-16T16:15:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742080_1256 (size=12001) 2024-11-16T16:15:15,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f38486fa14a04227ab0683d1757477cc 2024-11-16T16:15:15,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ae26508ba8c4479ea93e3f974f5d1bbf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf 2024-11-16T16:15:15,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T16:15:15,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/9890c63c69b44cc09b653685fdf55419 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419 2024-11-16T16:15:15,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T16:15:15,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f38486fa14a04227ab0683d1757477cc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc 2024-11-16T16:15:15,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc, entries=150, sequenceid=11, filesize=11.7 K 2024-11-16T16:15:15,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 1161dcb397ffa3f05e58f6711d26c4bb in 537ms, sequenceid=11, compaction requested=false 2024-11-16T16:15:15,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:15,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-16T16:15:15,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:15,891 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:15,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:15,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ec32546efad041b99003e08db4dd9b1d is 50, key is test_row_0/A:col10/1731773715336/Put/seqid=0 2024-11-16T16:15:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T16:15:15,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742081_1257 (size=12001) 2024-11-16T16:15:15,911 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ec32546efad041b99003e08db4dd9b1d 2024-11-16T16:15:15,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/03e30b2d36c644318d13c846ea01a0d5 is 50, key is test_row_0/B:col10/1731773715336/Put/seqid=0 2024-11-16T16:15:15,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742082_1258 (size=12001) 2024-11-16T16:15:15,927 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/03e30b2d36c644318d13c846ea01a0d5 2024-11-16T16:15:15,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/92d5b0f98d3844beb4b40e05c980230e is 50, key is test_row_0/C:col10/1731773715336/Put/seqid=0 2024-11-16T16:15:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742083_1259 (size=12001) 2024-11-16T16:15:15,967 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/92d5b0f98d3844beb4b40e05c980230e 2024-11-16T16:15:15,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ec32546efad041b99003e08db4dd9b1d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d 2024-11-16T16:15:15,980 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T16:15:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:15,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:15,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/03e30b2d36c644318d13c846ea01a0d5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5 2024-11-16T16:15:15,988 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T16:15:15,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/92d5b0f98d3844beb4b40e05c980230e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e 2024-11-16T16:15:15,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773775984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773775984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773775984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773775991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773775991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:15,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:15,996 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:15,999 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-16T16:15:16,000 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 1161dcb397ffa3f05e58f6711d26c4bb in 109ms, sequenceid=48, compaction requested=false 2024-11-16T16:15:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-16T16:15:16,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-16T16:15:16,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-16T16:15:16,005 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 726 msec 2024-11-16T16:15:16,008 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 735 msec 2024-11-16T16:15:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:16,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:16,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:16,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:16,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/6608ebc62b054828820008eacdc6e958 is 50, key is test_row_0/A:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:16,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742084_1260 (size=14341) 2024-11-16T16:15:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773776128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773776128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773776128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773776129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773776129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773776235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773776235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773776245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773776245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773776245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-16T16:15:16,415 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:16,416 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-16T16:15:16,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T16:15:16,417 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:16,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:16,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:16,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773776444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773776445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,449 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773776455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773776455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773776455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/6608ebc62b054828820008eacdc6e958 2024-11-16T16:15:16,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T16:15:16,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/01334e4a0e504315862855eafa8dd9da is 50, key is test_row_0/B:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:16,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742085_1261 (size=12001) 2024-11-16T16:15:16,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T16:15:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T16:15:16,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:16,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T16:15:16,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773776755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773776755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773776765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773776765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773776765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:16,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:16,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:15:16,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T16:15:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:16,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/01334e4a0e504315862855eafa8dd9da 2024-11-16T16:15:16,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/7d65e2abdbe34217acc9e15cc39bb298 is 50, key is test_row_0/C:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:16,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742086_1262 (size=12001) 2024-11-16T16:15:16,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/7d65e2abdbe34217acc9e15cc39bb298 2024-11-16T16:15:16,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/6608ebc62b054828820008eacdc6e958 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958 2024-11-16T16:15:16,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958, entries=200, sequenceid=61, filesize=14.0 K 2024-11-16T16:15:16,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/01334e4a0e504315862855eafa8dd9da as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da 2024-11-16T16:15:16,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:15:16,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/7d65e2abdbe34217acc9e15cc39bb298 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298 2024-11-16T16:15:16,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:15:16,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 1161dcb397ffa3f05e58f6711d26c4bb in 858ms, sequenceid=61, compaction requested=true 2024-11-16T16:15:16,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:16,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:16,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:16,965 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:16,965 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:16,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:16,967 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:16,967 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:16,967 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,967 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=37.4 K 2024-11-16T16:15:16,967 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:16,967 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:16,967 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:16,967 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae26508ba8c4479ea93e3f974f5d1bbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731773715281 2024-11-16T16:15:16,967 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.2 K 2024-11-16T16:15:16,968 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec32546efad041b99003e08db4dd9b1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731773715336 2024-11-16T16:15:16,968 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9890c63c69b44cc09b653685fdf55419, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731773715281 2024-11-16T16:15:16,968 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e30b2d36c644318d13c846ea01a0d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731773715336 2024-11-16T16:15:16,968 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6608ebc62b054828820008eacdc6e958, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:16,969 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 01334e4a0e504315862855eafa8dd9da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:16,988 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:16,989 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e3cbde42592048c09c24d8f1debe05ea is 50, key is test_row_0/A:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:16,991 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:16,992 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/31484bf9b3c341adbb0265c23bdea0b9 is 50, key is test_row_0/B:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:16,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742087_1263 (size=12104) 2024-11-16T16:15:17,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742088_1264 (size=12104) 2024-11-16T16:15:17,005 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e3cbde42592048c09c24d8f1debe05ea as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e3cbde42592048c09c24d8f1debe05ea 2024-11-16T16:15:17,011 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into e3cbde42592048c09c24d8f1debe05ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:17,011 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,011 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773716965; duration=0sec 2024-11-16T16:15:17,011 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:17,011 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:17,011 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:17,012 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:17,012 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:17,012 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,012 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.2 K 2024-11-16T16:15:17,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38486fa14a04227ab0683d1757477cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731773715281 2024-11-16T16:15:17,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92d5b0f98d3844beb4b40e05c980230e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731773715336 2024-11-16T16:15:17,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d65e2abdbe34217acc9e15cc39bb298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:17,019 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:17,019 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/3eaf9953e72243b9bea69298ef8be480 is 50, key is test_row_0/C:col10/1731773716106/Put/seqid=0 2024-11-16T16:15:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742089_1265 (size=12104) 2024-11-16T16:15:17,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,030 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:17,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/19c3b087d5804136a650640bd2f0a6ce is 50, key is test_row_0/A:col10/1731773716128/Put/seqid=0 2024-11-16T16:15:17,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742090_1266 (size=12001) 2024-11-16T16:15:17,038 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/19c3b087d5804136a650640bd2f0a6ce 2024-11-16T16:15:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T16:15:17,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3e0a7c7a68904e5399a5e7eaa32a4071 is 50, key is test_row_0/B:col10/1731773716128/Put/seqid=0 2024-11-16T16:15:17,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742091_1267 (size=12001) 2024-11-16T16:15:17,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:17,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773777276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773777279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773777280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773777281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773777282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773777385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773777395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773777395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773777395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773777395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,407 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/31484bf9b3c341adbb0265c23bdea0b9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/31484bf9b3c341adbb0265c23bdea0b9 2024-11-16T16:15:17,412 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 31484bf9b3c341adbb0265c23bdea0b9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:17,413 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,413 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773716965; duration=0sec 2024-11-16T16:15:17,413 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:17,413 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:17,427 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/3eaf9953e72243b9bea69298ef8be480 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/3eaf9953e72243b9bea69298ef8be480 2024-11-16T16:15:17,432 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 3eaf9953e72243b9bea69298ef8be480(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:17,432 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,432 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773716966; duration=0sec 2024-11-16T16:15:17,432 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:17,432 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:17,450 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3e0a7c7a68904e5399a5e7eaa32a4071 2024-11-16T16:15:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d39690898e2042f0b1119f1b746acb3b is 50, key is test_row_0/C:col10/1731773716128/Put/seqid=0 2024-11-16T16:15:17,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742092_1268 (size=12001) 2024-11-16T16:15:17,464 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d39690898e2042f0b1119f1b746acb3b 2024-11-16T16:15:17,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/19c3b087d5804136a650640bd2f0a6ce as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce 2024-11-16T16:15:17,474 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T16:15:17,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3e0a7c7a68904e5399a5e7eaa32a4071 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071 2024-11-16T16:15:17,479 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T16:15:17,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d39690898e2042f0b1119f1b746acb3b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b 2024-11-16T16:15:17,486 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b, entries=150, sequenceid=82, filesize=11.7 K 2024-11-16T16:15:17,487 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 1161dcb397ffa3f05e58f6711d26c4bb in 457ms, sequenceid=82, compaction requested=false 2024-11-16T16:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-16T16:15:17,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-16T16:15:17,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-16T16:15:17,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0710 sec 2024-11-16T16:15:17,492 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.0750 sec 2024-11-16T16:15:17,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-16T16:15:17,555 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:17,555 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-16T16:15:17,557 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:17,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:17,557 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:17,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:17,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:17,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:17,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e094b5762bae4b9987dfbf408666af1c is 50, key is test_row_0/A:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773777623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773777624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773777627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773777629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773777632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742093_1269 (size=12001) 2024-11-16T16:15:17,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e094b5762bae4b9987dfbf408666af1c 2024-11-16T16:15:17,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:17,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/72422285e3e545d195a1cc7cd51d1d4b is 50, key is test_row_0/B:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742094_1270 (size=12001) 2024-11-16T16:15:17,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/72422285e3e545d195a1cc7cd51d1d4b 2024-11-16T16:15:17,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/fbdcf6c1b5984301a3cbb4af0c1e0826 is 50, key is test_row_0/C:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742095_1271 (size=12001) 2024-11-16T16:15:17,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/fbdcf6c1b5984301a3cbb4af0c1e0826 2024-11-16T16:15:17,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e094b5762bae4b9987dfbf408666af1c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c 2024-11-16T16:15:17,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T16:15:17,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/72422285e3e545d195a1cc7cd51d1d4b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b 2024-11-16T16:15:17,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T16:15:17,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/fbdcf6c1b5984301a3cbb4af0c1e0826 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826 2024-11-16T16:15:17,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826, entries=150, sequenceid=98, filesize=11.7 K 2024-11-16T16:15:17,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1161dcb397ffa3f05e58f6711d26c4bb in 130ms, sequenceid=98, compaction requested=true 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:17,728 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:17,728 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:17,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:17,729 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:17,729 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:17,729 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:17,729 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:17,729 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,729 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,729 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e3cbde42592048c09c24d8f1debe05ea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.3 K 2024-11-16T16:15:17,729 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/31484bf9b3c341adbb0265c23bdea0b9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.3 K 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 31484bf9b3c341adbb0265c23bdea0b9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3cbde42592048c09c24d8f1debe05ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c3b087d5804136a650640bd2f0a6ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773716127 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e0a7c7a68904e5399a5e7eaa32a4071, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773716127 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e094b5762bae4b9987dfbf408666af1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:17,730 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 72422285e3e545d195a1cc7cd51d1d4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:17,738 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:17,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:17,754 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#227 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:17,754 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:17,755 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0688bac533494608afb2fae672a38bc0 is 50, key is test_row_0/A:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,755 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/bbbdace1ac044d1d87650f1aef8c2d19 is 50, key is test_row_0/B:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a3180ee649e645759435f1063c2d44bf is 50, key is test_row_0/A:col10/1731773717624/Put/seqid=0 2024-11-16T16:15:17,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773777749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773777750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773777758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773777759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773777761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742096_1272 (size=12207) 2024-11-16T16:15:17,790 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0688bac533494608afb2fae672a38bc0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0688bac533494608afb2fae672a38bc0 2024-11-16T16:15:17,799 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 0688bac533494608afb2fae672a38bc0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:17,800 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:17,800 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773717728; duration=0sec 2024-11-16T16:15:17,800 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:17,800 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:17,800 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:17,801 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:17,802 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:17,802 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,802 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/3eaf9953e72243b9bea69298ef8be480, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.3 K 2024-11-16T16:15:17,803 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eaf9953e72243b9bea69298ef8be480, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773715982 2024-11-16T16:15:17,804 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d39690898e2042f0b1119f1b746acb3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731773716127 2024-11-16T16:15:17,804 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbdcf6c1b5984301a3cbb4af0c1e0826, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:17,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742097_1273 (size=12207) 2024-11-16T16:15:17,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742098_1274 (size=14341) 2024-11-16T16:15:17,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a3180ee649e645759435f1063c2d44bf 2024-11-16T16:15:17,824 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:17,824 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/1ed19c903e794c8fbb95b34baa282c7b is 50, key is test_row_0/C:col10/1731773717275/Put/seqid=0 2024-11-16T16:15:17,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1106df3cf17d45709c4e317f3a6fb857 is 50, key is test_row_0/B:col10/1731773717624/Put/seqid=0 2024-11-16T16:15:17,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742099_1275 (size=12207) 2024-11-16T16:15:17,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742100_1276 (size=12001) 2024-11-16T16:15:17,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:17,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:17,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:17,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:17,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773777866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773777866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:17,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773777875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773777875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773777875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:17,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773778075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773778075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773778085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,089 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773778085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773778085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:18,212 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/bbbdace1ac044d1d87650f1aef8c2d19 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/bbbdace1ac044d1d87650f1aef8c2d19 2024-11-16T16:15:18,216 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into bbbdace1ac044d1d87650f1aef8c2d19(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:18,216 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:18,216 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773717728; duration=0sec 2024-11-16T16:15:18,216 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:18,216 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:18,253 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/1ed19c903e794c8fbb95b34baa282c7b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/1ed19c903e794c8fbb95b34baa282c7b 2024-11-16T16:15:18,257 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 1ed19c903e794c8fbb95b34baa282c7b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:18,257 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:18,257 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773717728; duration=0sec 2024-11-16T16:15:18,257 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:18,257 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:18,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1106df3cf17d45709c4e317f3a6fb857 2024-11-16T16:15:18,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/26e490f9e91e46f2a7c0dafb3781692a is 50, key is test_row_0/C:col10/1731773717624/Put/seqid=0 2024-11-16T16:15:18,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742101_1277 (size=12001) 2024-11-16T16:15:18,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773778385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773778385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773778395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773778395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773778395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:18,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/26e490f9e91e46f2a7c0dafb3781692a 2024-11-16T16:15:18,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a3180ee649e645759435f1063c2d44bf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf 2024-11-16T16:15:18,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf, entries=200, sequenceid=118, filesize=14.0 K 2024-11-16T16:15:18,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1106df3cf17d45709c4e317f3a6fb857 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857 2024-11-16T16:15:18,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T16:15:18,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/26e490f9e91e46f2a7c0dafb3781692a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a 2024-11-16T16:15:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:18,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-16T16:15:18,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1161dcb397ffa3f05e58f6711d26c4bb in 958ms, sequenceid=118, compaction requested=false 2024-11-16T16:15:18,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:18,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-16T16:15:18,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:18,782 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:18,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:18,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/2b8de78f272b494ea6e7a0757214ef2b is 50, key is test_row_0/A:col10/1731773717754/Put/seqid=0 2024-11-16T16:15:18,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742102_1278 (size=12051) 2024-11-16T16:15:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:18,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773778916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773778917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773718913, max timestamp=1731773718920Num uniques 859; 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773718913, max timestamp=1731773718920Num uniques 859; 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773718913, max timestamp=1731773718920Num uniques 859; 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:18,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:18,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773778946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773778948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773778949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:18,955 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773779025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773779025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773779055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773779065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773779065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773779235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773779235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,240 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/2b8de78f272b494ea6e7a0757214ef2b 2024-11-16T16:15:19,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/b006c3e85e434e0a88aea5bf11e2321d is 50, key is test_row_0/B:col10/1731773717754/Put/seqid=0 2024-11-16T16:15:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742103_1279 (size=12051) 2024-11-16T16:15:19,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773779265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773779275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773779275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773779545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773779545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773779575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773779585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773779585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:19,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:19,659 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/b006c3e85e434e0a88aea5bf11e2321d 2024-11-16T16:15:19,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/16536a1cfca847818f8725434f485a87 is 50, key is test_row_0/C:col10/1731773717754/Put/seqid=0 2024-11-16T16:15:19,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742104_1280 (size=12051) 2024-11-16T16:15:19,670 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/16536a1cfca847818f8725434f485a87 2024-11-16T16:15:19,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/2b8de78f272b494ea6e7a0757214ef2b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b 2024-11-16T16:15:19,681 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T16:15:19,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/b006c3e85e434e0a88aea5bf11e2321d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d 2024-11-16T16:15:19,688 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T16:15:19,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/16536a1cfca847818f8725434f485a87 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87 2024-11-16T16:15:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87, entries=150, sequenceid=133, filesize=11.8 K 2024-11-16T16:15:19,701 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 1161dcb397ffa3f05e58f6711d26c4bb in 919ms, sequenceid=133, compaction requested=true 2024-11-16T16:15:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:19,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-16T16:15:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-16T16:15:19,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-16T16:15:19,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1450 sec 2024-11-16T16:15:19,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:19,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.1500 sec 2024-11-16T16:15:20,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:20,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:20,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:20,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8123e36e754c4a0488edc6884e07fb6b is 50, key is test_row_0/A:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:20,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773780085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773780085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773780087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773780095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773780095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742105_1281 (size=12151) 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773780205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773780205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773780415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773780426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8123e36e754c4a0488edc6884e07fb6b 2024-11-16T16:15:20,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3ecd732fb29a4aff9a5764e5015cde14 is 50, key is test_row_0/B:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:20,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742106_1282 (size=12151) 2024-11-16T16:15:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773780725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:20,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773780735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:20,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:20,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3ecd732fb29a4aff9a5764e5015cde14 2024-11-16T16:15:20,966 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/03a924faabca4c5080530a25ced133c7 is 50, key is test_row_0/C:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:20,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742107_1283 (size=12151) 2024-11-16T16:15:20,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/03a924faabca4c5080530a25ced133c7 2024-11-16T16:15:20,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8123e36e754c4a0488edc6884e07fb6b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b 2024-11-16T16:15:20,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T16:15:20,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/3ecd732fb29a4aff9a5764e5015cde14 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14 2024-11-16T16:15:20,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T16:15:20,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/03a924faabca4c5080530a25ced133c7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7 2024-11-16T16:15:21,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7, entries=150, sequenceid=167, filesize=11.9 K 2024-11-16T16:15:21,003 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 1161dcb397ffa3f05e58f6711d26c4bb in 944ms, sequenceid=167, compaction requested=true 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:21,003 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:21,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:21,003 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:21,005 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:21,005 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:21,005 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:21,005 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:21,005 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0688bac533494608afb2fae672a38bc0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=49.6 K 2024-11-16T16:15:21,005 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:21,005 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:21,005 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/bbbdace1ac044d1d87650f1aef8c2d19, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=47.3 K 2024-11-16T16:15:21,006 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0688bac533494608afb2fae672a38bc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:21,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting bbbdace1ac044d1d87650f1aef8c2d19, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:21,006 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3180ee649e645759435f1063c2d44bf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731773717624 2024-11-16T16:15:21,007 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1106df3cf17d45709c4e317f3a6fb857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731773717624 2024-11-16T16:15:21,007 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b8de78f272b494ea6e7a0757214ef2b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731773717753 2024-11-16T16:15:21,007 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b006c3e85e434e0a88aea5bf11e2321d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731773717753 2024-11-16T16:15:21,007 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8123e36e754c4a0488edc6884e07fb6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:21,008 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ecd732fb29a4aff9a5764e5015cde14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:21,022 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#239 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:21,022 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/acf44847a0464730b77cc0936bc35b9e is 50, key is test_row_0/A:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:21,031 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:21,032 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/efaab7eb026d4179b0a72ec1829fd25d is 50, key is test_row_0/B:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:21,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742109_1285 (size=12493) 2024-11-16T16:15:21,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742108_1284 (size=12493) 2024-11-16T16:15:21,057 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/efaab7eb026d4179b0a72ec1829fd25d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/efaab7eb026d4179b0a72ec1829fd25d 2024-11-16T16:15:21,062 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into efaab7eb026d4179b0a72ec1829fd25d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:21,063 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:21,063 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=12, startTime=1731773721003; duration=0sec 2024-11-16T16:15:21,063 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:21,063 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:21,063 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:21,064 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:21,064 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:21,064 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:21,064 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/1ed19c903e794c8fbb95b34baa282c7b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=47.3 K 2024-11-16T16:15:21,064 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ed19c903e794c8fbb95b34baa282c7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731773717275 2024-11-16T16:15:21,065 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 26e490f9e91e46f2a7c0dafb3781692a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731773717624 2024-11-16T16:15:21,065 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 16536a1cfca847818f8725434f485a87, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731773717753 2024-11-16T16:15:21,065 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 03a924faabca4c5080530a25ced133c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:21,076 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:21,076 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/516fa0010177422780fe8936272cd640 is 50, key is test_row_0/C:col10/1731773718948/Put/seqid=0 2024-11-16T16:15:21,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742110_1286 (size=12493) 2024-11-16T16:15:21,100 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/516fa0010177422780fe8936272cd640 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/516fa0010177422780fe8936272cd640 2024-11-16T16:15:21,106 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 516fa0010177422780fe8936272cd640(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:21,106 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:21,106 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=12, startTime=1731773721003; duration=0sec 2024-11-16T16:15:21,106 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:21,107 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:21,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:21,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:21,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/013d3bf35bb64daeb1c23dc74ae3b7b3 is 50, key is test_row_0/A:col10/1731773720085/Put/seqid=0 2024-11-16T16:15:21,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742111_1287 (size=16931) 2024-11-16T16:15:21,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/013d3bf35bb64daeb1c23dc74ae3b7b3 2024-11-16T16:15:21,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/ac134cb35e884da59188384cfd0d6125 is 50, key is test_row_0/B:col10/1731773720085/Put/seqid=0 2024-11-16T16:15:21,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773781180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773781182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742112_1288 (size=12151) 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773781182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/ac134cb35e884da59188384cfd0d6125 2024-11-16T16:15:21,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/05335233a5ed421a93d456bff7d81f09 is 50, key is test_row_0/C:col10/1731773720085/Put/seqid=0 2024-11-16T16:15:21,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742113_1289 (size=12151) 2024-11-16T16:15:21,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/05335233a5ed421a93d456bff7d81f09 2024-11-16T16:15:21,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/013d3bf35bb64daeb1c23dc74ae3b7b3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3 2024-11-16T16:15:21,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3, entries=250, sequenceid=182, filesize=16.5 K 2024-11-16T16:15:21,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/ac134cb35e884da59188384cfd0d6125 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125 2024-11-16T16:15:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773781234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773781245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T16:15:21,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/05335233a5ed421a93d456bff7d81f09 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09 2024-11-16T16:15:21,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T16:15:21,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 1161dcb397ffa3f05e58f6711d26c4bb in 148ms, sequenceid=182, compaction requested=false 2024-11-16T16:15:21,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:21,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T16:15:21,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773721150, max timestamp=1731773721300Num uniques 859; 2024-11-16T16:15:21,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773721150, max timestamp=1731773721300Num uniques 859; 2024-11-16T16:15:21,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:21,303 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:21,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731773721150, max timestamp=1731773721300Num uniques 859; 2024-11-16T16:15:21,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:21,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:21,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/1286e7f4f7c747f7a17c6ad1d6f98247 is 50, key is test_row_0/A:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:21,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742114_1290 (size=12151) 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773721351, max timestamp=1731773721365Num uniques 859; 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773721351, max timestamp=1731773721365Num uniques 859; 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773721351, max timestamp=1731773721365Num uniques 859; 2024-11-16T16:15:21,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731773721386, max timestamp=1731773721429Num uniques 1068; 2024-11-16T16:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731773721386, max timestamp=1731773721429Num uniques 1068; 2024-11-16T16:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T16:15:21,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T16:15:21,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731773721386, max timestamp=1731773721429Num uniques 1068; 2024-11-16T16:15:21,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-16T16:15:21,453 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/acf44847a0464730b77cc0936bc35b9e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/acf44847a0464730b77cc0936bc35b9e 2024-11-16T16:15:21,460 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into acf44847a0464730b77cc0936bc35b9e(size=12.2 K), total size for store is 28.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:21,460 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:21,460 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=12, startTime=1731773721003; duration=0sec 2024-11-16T16:15:21,460 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:21,460 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:21,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731773721461, max timestamp=1731773721485Num uniques 1027; 2024-11-16T16:15:21,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-16T16:15:21,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731773721461, max timestamp=1731773721485Num uniques 1027; 2024-11-16T16:15:21,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-16T16:15:21,491 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731773721461, max timestamp=1731773721485Num uniques 1027; 2024-11-16T16:15:21,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-16T16:15:21,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731773721512, max timestamp=1731773721533Num uniques 1036; 2024-11-16T16:15:21,537 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-16T16:15:21,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731773721512, max timestamp=1731773721533Num uniques 1036; 2024-11-16T16:15:21,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731773721512, max timestamp=1731773721533Num uniques 1036; 2024-11-16T16:15:21,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-16T16:15:21,538 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=13096, suffix heap size=135712,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=286 2024-11-16T16:15:21,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773781580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773781582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773781583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773781695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773781695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773781705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-16T16:15:21,715 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:21,715 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-16T16:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:21,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:21,717 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:21,717 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:21,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/1286e7f4f7c747f7a17c6ad1d6f98247 2024-11-16T16:15:21,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1ce8d49a56a9411cac5232668fd62a07 is 50, key is test_row_0/B:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:21,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742115_1291 (size=12151) 2024-11-16T16:15:21,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1ce8d49a56a9411cac5232668fd62a07 2024-11-16T16:15:21,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d960496932ba44cbb87ac2811806e454 is 50, key is test_row_0/C:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:21,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742116_1292 (size=12151) 2024-11-16T16:15:21,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:21,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:21,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:21,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:21,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773781905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773781905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:21,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773781915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:21,922 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:22,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:15:22,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T16:15:22,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.02 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d960496932ba44cbb87ac2811806e454 2024-11-16T16:15:22,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/1286e7f4f7c747f7a17c6ad1d6f98247 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247 2024-11-16T16:15:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773782215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,219 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773782216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T16:15:22,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/1ce8d49a56a9411cac5232668fd62a07 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07 2024-11-16T16:15:22,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T16:15:22,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d960496932ba44cbb87ac2811806e454 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454 2024-11-16T16:15:22,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773782225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454, entries=150, sequenceid=205, filesize=11.9 K 2024-11-16T16:15:22,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.86 KB/40815, heapSize ~105.12 KB/107640, currentSize=147.60 KB/151140 for 1161dcb397ffa3f05e58f6711d26c4bb in 932ms, sequenceid=205, compaction requested=true 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:22,234 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:22,234 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:22,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:22,236 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41575 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:22,236 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:22,236 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:22,236 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:22,236 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,236 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,237 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/efaab7eb026d4179b0a72ec1829fd25d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.9 K 2024-11-16T16:15:22,237 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/acf44847a0464730b77cc0936bc35b9e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=40.6 K 2024-11-16T16:15:22,237 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting efaab7eb026d4179b0a72ec1829fd25d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:22,237 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting acf44847a0464730b77cc0936bc35b9e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:22,237 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ac134cb35e884da59188384cfd0d6125, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773720085 2024-11-16T16:15:22,237 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013d3bf35bb64daeb1c23dc74ae3b7b3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773720060 2024-11-16T16:15:22,238 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce8d49a56a9411cac5232668fd62a07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:22,238 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1286e7f4f7c747f7a17c6ad1d6f98247, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:22,246 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:22,247 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/6b0d008713e14d8a8bd8aa7beb8d9ff0 is 50, key is test_row_0/A:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:22,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731773721568, max timestamp=1731773722247Num uniques 1145; 2024-11-16T16:15:22,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731773721568, max timestamp=1731773722247Num uniques 1145; 2024-11-16T16:15:22,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-16T16:15:22,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-16T16:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:22,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=69.33 KB heapSize=186.42 KB 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:22,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:15:22,254 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:22,255 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/77d24687645c4cfd9d113b36e44f7f1c is 50, key is test_row_0/B:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742117_1293 (size=12595) 2024-11-16T16:15:22,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b139e36e3faf4b1b8cb4c802bce9e95e is 50, key is test_row_0/A:col10/1731773722247/Put/seqid=0 2024-11-16T16:15:22,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742118_1294 (size=12595) 2024-11-16T16:15:22,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742119_1295 (size=14741) 2024-11-16T16:15:22,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b139e36e3faf4b1b8cb4c802bce9e95e 2024-11-16T16:15:22,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c27e4650b2a14907878b61f5738b8e95 is 50, key is test_row_0/B:col10/1731773722247/Put/seqid=0 2024-11-16T16:15:22,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742120_1296 (size=12301) 2024-11-16T16:15:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773782305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773782305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773782415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773782415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773782625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773782625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,630 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:22,670 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/6b0d008713e14d8a8bd8aa7beb8d9ff0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6b0d008713e14d8a8bd8aa7beb8d9ff0 2024-11-16T16:15:22,676 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 6b0d008713e14d8a8bd8aa7beb8d9ff0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:22,676 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:22,676 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773722234; duration=0sec 2024-11-16T16:15:22,676 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:22,676 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:22,676 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:22,677 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:22,677 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:22,677 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,677 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/516fa0010177422780fe8936272cd640, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=35.9 K 2024-11-16T16:15:22,677 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 516fa0010177422780fe8936272cd640, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731773718944 2024-11-16T16:15:22,678 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05335233a5ed421a93d456bff7d81f09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773720085 2024-11-16T16:15:22,678 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d960496932ba44cbb87ac2811806e454, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:22,682 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/77d24687645c4cfd9d113b36e44f7f1c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/77d24687645c4cfd9d113b36e44f7f1c 2024-11-16T16:15:22,688 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 77d24687645c4cfd9d113b36e44f7f1c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:22,688 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:22,688 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773722234; duration=0sec 2024-11-16T16:15:22,688 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:22,688 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:22,688 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:22,689 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b66d53e66a0146629508ad9979903b55 is 50, key is test_row_0/C:col10/1731773721301/Put/seqid=0 2024-11-16T16:15:22,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c27e4650b2a14907878b61f5738b8e95 2024-11-16T16:15:22,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742121_1297 (size=12595) 2024-11-16T16:15:22,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/6007e88ca0b84b5ea1bff572c06d2d15 is 50, key is test_row_0/C:col10/1731773722247/Put/seqid=0 2024-11-16T16:15:22,709 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b66d53e66a0146629508ad9979903b55 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b66d53e66a0146629508ad9979903b55 2024-11-16T16:15:22,716 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into b66d53e66a0146629508ad9979903b55(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:22,716 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:22,716 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773722234; duration=0sec 2024-11-16T16:15:22,716 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:22,716 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:22,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773782725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773782726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742122_1298 (size=12301) 2024-11-16T16:15:22,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/6007e88ca0b84b5ea1bff572c06d2d15 2024-11-16T16:15:22,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773782735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b139e36e3faf4b1b8cb4c802bce9e95e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e 2024-11-16T16:15:22,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e, entries=200, sequenceid=295, filesize=14.4 K 2024-11-16T16:15:22,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c27e4650b2a14907878b61f5738b8e95 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95 2024-11-16T16:15:22,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T16:15:22,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/6007e88ca0b84b5ea1bff572c06d2d15 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15 2024-11-16T16:15:22,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15, entries=150, sequenceid=295, filesize=12.0 K 2024-11-16T16:15:22,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~186.38 KB/190848, currentSize=114.05 KB/116790 for 1161dcb397ffa3f05e58f6711d26c4bb in 511ms, sequenceid=295, compaction requested=false 2024-11-16T16:15:22,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:22,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-16T16:15:22,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:22,788 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:22,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:22,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8701178717f84df49ea4f5ef43d230cc is 50, key is test_row_0/A:col10/1731773722292/Put/seqid=0 2024-11-16T16:15:22,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742123_1299 (size=12301) 2024-11-16T16:15:22,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:22,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:22,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773782959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:22,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773782962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:22,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773783065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773783075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,202 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8701178717f84df49ea4f5ef43d230cc 2024-11-16T16:15:23,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/5b72e1e968d94614a33dad1705644680 is 50, key is test_row_0/B:col10/1731773722292/Put/seqid=0 2024-11-16T16:15:23,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742124_1300 (size=12301) 2024-11-16T16:15:23,230 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/5b72e1e968d94614a33dad1705644680 2024-11-16T16:15:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/68cd7070a4e845218a3a5a31b37b7f15 is 50, key is test_row_0/C:col10/1731773722292/Put/seqid=0 2024-11-16T16:15:23,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742125_1301 (size=12301) 2024-11-16T16:15:23,258 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/68cd7070a4e845218a3a5a31b37b7f15 2024-11-16T16:15:23,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8701178717f84df49ea4f5ef43d230cc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc 2024-11-16T16:15:23,272 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T16:15:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/5b72e1e968d94614a33dad1705644680 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680 2024-11-16T16:15:23,279 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T16:15:23,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773783275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/68cd7070a4e845218a3a5a31b37b7f15 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15 2024-11-16T16:15:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773783285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,287 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15, entries=150, sequenceid=318, filesize=12.0 K 2024-11-16T16:15:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,288 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1161dcb397ffa3f05e58f6711d26c4bb in 500ms, sequenceid=318, compaction requested=true 2024-11-16T16:15:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-16T16:15:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-16T16:15:23,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-16T16:15:23,292 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5720 sec 2024-11-16T16:15:23,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.5790 sec 2024-11-16T16:15:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:23,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:23,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:23,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/11191a59b7464f6889eb65741bccc573 is 50, key is test_row_0/A:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:23,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742126_1302 (size=14741) 2024-11-16T16:15:23,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773783636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,642 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773783641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773783744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773783745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773783745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,748 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:23,748 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773783755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773783755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,759 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-16T16:15:23,865 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:23,866 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-16T16:15:23,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T16:15:23,868 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:23,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:23,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:23,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773783955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773783965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:23,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T16:15:24,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/11191a59b7464f6889eb65741bccc573 2024-11-16T16:15:24,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87c75a57362b4b1194b58c9459db585b is 50, key is test_row_0/B:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:24,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T16:15:24,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:24,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742127_1303 (size=12301) 2024-11-16T16:15:24,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T16:15:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T16:15:24,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773784265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:24,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773784275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:24,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T16:15:24,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87c75a57362b4b1194b58c9459db585b 2024-11-16T16:15:24,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/12f47407d4e44ee9a87c86c66795996e is 50, key is test_row_0/C:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:24,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T16:15:24,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T16:15:24,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742128_1304 (size=12301) 2024-11-16T16:15:24,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/12f47407d4e44ee9a87c86c66795996e 2024-11-16T16:15:24,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/11191a59b7464f6889eb65741bccc573 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573 2024-11-16T16:15:24,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573, entries=200, sequenceid=332, filesize=14.4 K 2024-11-16T16:15:24,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87c75a57362b4b1194b58c9459db585b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b 2024-11-16T16:15:24,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b, entries=150, sequenceid=332, filesize=12.0 K 2024-11-16T16:15:24,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/12f47407d4e44ee9a87c86c66795996e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e 2024-11-16T16:15:24,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e, entries=150, sequenceid=332, filesize=12.0 K 2024-11-16T16:15:24,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 1161dcb397ffa3f05e58f6711d26c4bb in 961ms, sequenceid=332, compaction requested=true 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:24,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-16T16:15:24,559 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:24,559 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:24,561 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:24,561 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:24,561 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,561 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b66d53e66a0146629508ad9979903b55, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=48.3 K 2024-11-16T16:15:24,561 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54378 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:24,561 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:24,561 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,561 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6b0d008713e14d8a8bd8aa7beb8d9ff0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=53.1 K 2024-11-16T16:15:24,562 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b0d008713e14d8a8bd8aa7beb8d9ff0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:24,563 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b66d53e66a0146629508ad9979903b55, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:24,563 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b139e36e3faf4b1b8cb4c802bce9e95e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731773721568 2024-11-16T16:15:24,563 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6007e88ca0b84b5ea1bff572c06d2d15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731773721568 2024-11-16T16:15:24,563 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8701178717f84df49ea4f5ef43d230cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731773722292 2024-11-16T16:15:24,564 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 68cd7070a4e845218a3a5a31b37b7f15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731773722292 2024-11-16T16:15:24,564 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11191a59b7464f6889eb65741bccc573, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:24,564 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 12f47407d4e44ee9a87c86c66795996e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:24,591 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#260 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:24,592 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/21c9ae6f80b84985870e1c11dac574f0 is 50, key is test_row_0/C:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:24,603 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:24,604 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8d73e29b1013454c9694bf723c694edb is 50, key is test_row_0/A:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:24,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-16T16:15:24,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,638 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:24,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742130_1306 (size=12881) 2024-11-16T16:15:24,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742129_1305 (size=12881) 2024-11-16T16:15:24,701 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8d73e29b1013454c9694bf723c694edb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8d73e29b1013454c9694bf723c694edb 2024-11-16T16:15:24,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/70c22e6d33e14f489e81e14ff4738cb3 is 50, key is test_row_0/A:col10/1731773723629/Put/seqid=0 2024-11-16T16:15:24,713 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/21c9ae6f80b84985870e1c11dac574f0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/21c9ae6f80b84985870e1c11dac574f0 2024-11-16T16:15:24,713 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 8d73e29b1013454c9694bf723c694edb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:24,713 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:24,714 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=12, startTime=1731773724559; duration=0sec 2024-11-16T16:15:24,714 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:24,714 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:24,714 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:24,716 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49498 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:24,717 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:24,717 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,717 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/77d24687645c4cfd9d113b36e44f7f1c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=48.3 K 2024-11-16T16:15:24,718 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77d24687645c4cfd9d113b36e44f7f1c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731773721180 2024-11-16T16:15:24,718 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c27e4650b2a14907878b61f5738b8e95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731773721568 2024-11-16T16:15:24,720 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b72e1e968d94614a33dad1705644680, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731773722292 2024-11-16T16:15:24,721 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c75a57362b4b1194b58c9459db585b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:24,724 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 21c9ae6f80b84985870e1c11dac574f0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:24,724 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:24,724 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=12, startTime=1731773724559; duration=0sec 2024-11-16T16:15:24,724 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:24,724 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:24,743 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#263 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:24,744 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/529b57bf376548cea7e591c75cd0e04b is 50, key is test_row_0/B:col10/1731773723598/Put/seqid=0 2024-11-16T16:15:24,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742131_1307 (size=12301) 2024-11-16T16:15:24,749 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/70c22e6d33e14f489e81e14ff4738cb3 2024-11-16T16:15:24,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7e139999b8b049f1802ea654c5a3f68c is 50, key is test_row_0/B:col10/1731773723629/Put/seqid=0 2024-11-16T16:15:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:24,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742132_1308 (size=12881) 2024-11-16T16:15:24,808 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/529b57bf376548cea7e591c75cd0e04b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/529b57bf376548cea7e591c75cd0e04b 2024-11-16T16:15:24,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742133_1309 (size=12301) 2024-11-16T16:15:24,815 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7e139999b8b049f1802ea654c5a3f68c 2024-11-16T16:15:24,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773784811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:24,819 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 529b57bf376548cea7e591c75cd0e04b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:24,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:24,819 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=12, startTime=1731773724559; duration=0sec 2024-11-16T16:15:24,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:24,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773784817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:24,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b0bee738fd32441586ec9130ac4bda5a is 50, key is test_row_0/C:col10/1731773723629/Put/seqid=0 2024-11-16T16:15:24,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742134_1310 (size=12301) 2024-11-16T16:15:24,854 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b0bee738fd32441586ec9130ac4bda5a 2024-11-16T16:15:24,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/70c22e6d33e14f489e81e14ff4738cb3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3 2024-11-16T16:15:24,864 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3, entries=150, sequenceid=350, filesize=12.0 K 2024-11-16T16:15:24,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7e139999b8b049f1802ea654c5a3f68c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c 2024-11-16T16:15:24,869 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c, entries=150, sequenceid=350, filesize=12.0 K 2024-11-16T16:15:24,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b0bee738fd32441586ec9130ac4bda5a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a 2024-11-16T16:15:24,874 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a, entries=150, sequenceid=350, filesize=12.0 K 2024-11-16T16:15:24,874 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1161dcb397ffa3f05e58f6711d26c4bb in 237ms, sequenceid=350, compaction requested=false 2024-11-16T16:15:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:24,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-16T16:15:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-16T16:15:24,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-16T16:15:24,877 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0070 sec 2024-11-16T16:15:24,880 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.0120 sec 2024-11-16T16:15:24,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:24,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:24,928 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:24,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b7ce42302e74ebf9054b14035cf09f8 is 50, key is test_row_0/A:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:24,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742135_1311 (size=14741) 2024-11-16T16:15:24,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773784962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:24,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773784967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:24,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-16T16:15:25,005 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:25,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-16T16:15:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:25,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:25,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:25,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773785075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773785075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:25,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773785285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773785285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:25,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b7ce42302e74ebf9054b14035cf09f8 2024-11-16T16:15:25,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/fdf8f3ffb07a4dae91e6ce53f05599ea is 50, key is test_row_0/B:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:25,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742136_1312 (size=12301) 2024-11-16T16:15:25,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773785594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773785595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:25,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/fdf8f3ffb07a4dae91e6ce53f05599ea 2024-11-16T16:15:25,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773785776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773785776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,777 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,777 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773785776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:25,779 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:25,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:25,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/86620cd99baf484fb49a05f29512df3c is 50, key is test_row_0/C:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:25,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742137_1313 (size=12301) 2024-11-16T16:15:25,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:25,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:26,077 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:26,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773786105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:26,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773786105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:26,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:26,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/86620cd99baf484fb49a05f29512df3c 2024-11-16T16:15:26,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b7ce42302e74ebf9054b14035cf09f8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8 2024-11-16T16:15:26,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8, entries=200, sequenceid=368, filesize=14.4 K 2024-11-16T16:15:26,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/fdf8f3ffb07a4dae91e6ce53f05599ea as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea 2024-11-16T16:15:26,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T16:15:26,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/86620cd99baf484fb49a05f29512df3c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c 2024-11-16T16:15:26,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c, entries=150, sequenceid=368, filesize=12.0 K 2024-11-16T16:15:26,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1161dcb397ffa3f05e58f6711d26c4bb in 1297ms, sequenceid=368, compaction requested=true 2024-11-16T16:15:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:26,224 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:26,225 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:26,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:26,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:26,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:26,225 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:26,226 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:26,226 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:26,226 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:26,226 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:26,226 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,226 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/529b57bf376548cea7e591c75cd0e04b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.6 K 2024-11-16T16:15:26,226 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,226 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8d73e29b1013454c9694bf723c694edb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=39.0 K 2024-11-16T16:15:26,227 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 529b57bf376548cea7e591c75cd0e04b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:26,227 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d73e29b1013454c9694bf723c694edb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:26,227 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e139999b8b049f1802ea654c5a3f68c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731773723629 2024-11-16T16:15:26,227 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c22e6d33e14f489e81e14ff4738cb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731773723629 2024-11-16T16:15:26,227 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting fdf8f3ffb07a4dae91e6ce53f05599ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:26,228 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b7ce42302e74ebf9054b14035cf09f8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:26,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-16T16:15:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,231 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:26,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:26,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0d24fc2c61734f78aef0cbc32d1dcc71 is 50, key is test_row_0/A:col10/1731773724961/Put/seqid=0 2024-11-16T16:15:26,250 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:26,251 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7706913c6a034913b229ff9ffdd3814f is 50, key is test_row_0/B:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:26,259 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#271 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:26,259 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/470cbc7e40504d3d8aec97de2599b439 is 50, key is test_row_0/A:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:26,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742138_1314 (size=12301) 2024-11-16T16:15:26,297 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0d24fc2c61734f78aef0cbc32d1dcc71 2024-11-16T16:15:26,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742139_1315 (size=12983) 2024-11-16T16:15:26,322 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7706913c6a034913b229ff9ffdd3814f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7706913c6a034913b229ff9ffdd3814f 2024-11-16T16:15:26,330 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 7706913c6a034913b229ff9ffdd3814f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:26,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:26,330 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773726224; duration=0sec 2024-11-16T16:15:26,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:26,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:26,330 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:26,332 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:26,332 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:26,332 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,332 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/21c9ae6f80b84985870e1c11dac574f0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.6 K 2024-11-16T16:15:26,333 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c9ae6f80b84985870e1c11dac574f0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731773722957 2024-11-16T16:15:26,333 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b0bee738fd32441586ec9130ac4bda5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1731773723629 2024-11-16T16:15:26,333 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 86620cd99baf484fb49a05f29512df3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:26,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/813978114b034923b645f529896648d4 is 50, key is test_row_0/B:col10/1731773724961/Put/seqid=0 2024-11-16T16:15:26,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742140_1316 (size=12983) 2024-11-16T16:15:26,349 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/470cbc7e40504d3d8aec97de2599b439 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/470cbc7e40504d3d8aec97de2599b439 2024-11-16T16:15:26,354 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:26,355 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f56279568c4943c395b30658bb3c0b16 is 50, key is test_row_0/C:col10/1731773724804/Put/seqid=0 2024-11-16T16:15:26,361 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 470cbc7e40504d3d8aec97de2599b439(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:26,361 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:26,361 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773726224; duration=0sec 2024-11-16T16:15:26,361 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:26,361 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:26,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742141_1317 (size=12301) 2024-11-16T16:15:26,362 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/813978114b034923b645f529896648d4 2024-11-16T16:15:26,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/664a1b2a20a644f995210cf7d71932c5 is 50, key is test_row_0/C:col10/1731773724961/Put/seqid=0 2024-11-16T16:15:26,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742142_1318 (size=12983) 2024-11-16T16:15:26,385 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f56279568c4943c395b30658bb3c0b16 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f56279568c4943c395b30658bb3c0b16 2024-11-16T16:15:26,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742143_1319 (size=12301) 2024-11-16T16:15:26,391 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into f56279568c4943c395b30658bb3c0b16(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:26,391 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:26,391 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773726225; duration=0sec 2024-11-16T16:15:26,391 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:26,391 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:26,788 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/664a1b2a20a644f995210cf7d71932c5 2024-11-16T16:15:26,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0d24fc2c61734f78aef0cbc32d1dcc71 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71 2024-11-16T16:15:26,810 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T16:15:26,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/813978114b034923b645f529896648d4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4 2024-11-16T16:15:26,821 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T16:15:26,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/664a1b2a20a644f995210cf7d71932c5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5 2024-11-16T16:15:26,826 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5, entries=150, sequenceid=385, filesize=12.0 K 2024-11-16T16:15:26,828 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for 1161dcb397ffa3f05e58f6711d26c4bb in 598ms, sequenceid=385, compaction requested=false 2024-11-16T16:15:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-16T16:15:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-16T16:15:26,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-16T16:15:26,832 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-16T16:15:26,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8280 sec 2024-11-16T16:15:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-16T16:15:27,155 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:27,156 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-16T16:15:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:27,159 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:27,160 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:27,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:27,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:27,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:27,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:27,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b446dac100fd4c95894e880bd3d22496 is 50, key is test_row_0/A:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:27,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742144_1320 (size=17181) 2024-11-16T16:15:27,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b446dac100fd4c95894e880bd3d22496 2024-11-16T16:15:27,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87fc5c3d4d7f45b986bf712074f28b69 is 50, key is test_row_0/B:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742145_1321 (size=12301) 2024-11-16T16:15:27,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87fc5c3d4d7f45b986bf712074f28b69 2024-11-16T16:15:27,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/40d3cb907ba649ee9b7b8ebda9297fe6 is 50, key is test_row_0/C:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:27,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773727207, max timestamp=1731773727224Num uniques 859; 2024-11-16T16:15:27,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:27,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773727207, max timestamp=1731773727224Num uniques 859; 2024-11-16T16:15:27,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:27,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773727207, max timestamp=1731773727224Num uniques 859; 2024-11-16T16:15:27,234 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:27,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742146_1322 (size=12301) 2024-11-16T16:15:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:27,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773787289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773787289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T16:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773787405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773787405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T16:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773787615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773787615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T16:15:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:27,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/40d3cb907ba649ee9b7b8ebda9297fe6 2024-11-16T16:15:27,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/b446dac100fd4c95894e880bd3d22496 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496 2024-11-16T16:15:27,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496, entries=250, sequenceid=398, filesize=16.8 K 2024-11-16T16:15:27,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/87fc5c3d4d7f45b986bf712074f28b69 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69 2024-11-16T16:15:27,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69, entries=150, sequenceid=398, filesize=12.0 K 2024-11-16T16:15:27,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/40d3cb907ba649ee9b7b8ebda9297fe6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6 2024-11-16T16:15:27,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6, entries=150, sequenceid=398, filesize=12.0 K 2024-11-16T16:15:27,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 1161dcb397ffa3f05e58f6711d26c4bb in 501ms, sequenceid=398, compaction requested=true 2024-11-16T16:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:27,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:27,663 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:27,664 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:27,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:27,664 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:27,664 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:27,664 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:27,664 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:27,664 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,664 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,665 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/470cbc7e40504d3d8aec97de2599b439, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=41.5 K 2024-11-16T16:15:27,665 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7706913c6a034913b229ff9ffdd3814f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.7 K 2024-11-16T16:15:27,665 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7706913c6a034913b229ff9ffdd3814f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:27,665 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 470cbc7e40504d3d8aec97de2599b439, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:27,666 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d24fc2c61734f78aef0cbc32d1dcc71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731773724946 2024-11-16T16:15:27,666 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 813978114b034923b645f529896648d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731773724946 2024-11-16T16:15:27,666 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b446dac100fd4c95894e880bd3d22496, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727115 2024-11-16T16:15:27,666 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 87fc5c3d4d7f45b986bf712074f28b69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727127 2024-11-16T16:15:27,674 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:27,674 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/95fc6851ae6c4e4fb371fb7b703d3a53 is 50, key is test_row_0/B:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:27,682 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:27,682 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ca86b65cf0984a05ad3f285c23247ebf is 50, key is test_row_0/A:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:27,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742147_1323 (size=13085) 2024-11-16T16:15:27,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742148_1324 (size=13085) 2024-11-16T16:15:27,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:27,774 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:27,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:27,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0640097e275b4afcb43e08062de112fb is 50, key is test_row_0/A:col10/1731773727287/Put/seqid=0 2024-11-16T16:15:27,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742149_1325 (size=12301) 2024-11-16T16:15:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:27,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773787948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773787948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:27,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773788065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773788065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,093 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/95fc6851ae6c4e4fb371fb7b703d3a53 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/95fc6851ae6c4e4fb371fb7b703d3a53 2024-11-16T16:15:28,097 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/ca86b65cf0984a05ad3f285c23247ebf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ca86b65cf0984a05ad3f285c23247ebf 2024-11-16T16:15:28,097 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 95fc6851ae6c4e4fb371fb7b703d3a53(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:28,097 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:28,097 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773727663; duration=0sec 2024-11-16T16:15:28,097 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:28,097 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:28,097 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:28,099 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:28,099 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:28,099 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:28,099 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f56279568c4943c395b30658bb3c0b16, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.7 K 2024-11-16T16:15:28,099 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f56279568c4943c395b30658bb3c0b16, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731773724804 2024-11-16T16:15:28,099 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 664a1b2a20a644f995210cf7d71932c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731773724946 2024-11-16T16:15:28,100 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 40d3cb907ba649ee9b7b8ebda9297fe6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727127 2024-11-16T16:15:28,103 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into ca86b65cf0984a05ad3f285c23247ebf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:28,103 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:28,103 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773727663; duration=0sec 2024-11-16T16:15:28,103 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:28,103 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:28,107 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:28,108 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f76086613fd04414ba800c5ef85e0bd3 is 50, key is test_row_0/C:col10/1731773727161/Put/seqid=0 2024-11-16T16:15:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742150_1326 (size=13085) 2024-11-16T16:15:28,122 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/f76086613fd04414ba800c5ef85e0bd3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f76086613fd04414ba800c5ef85e0bd3 2024-11-16T16:15:28,127 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into f76086613fd04414ba800c5ef85e0bd3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:28,127 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:28,127 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773727664; duration=0sec 2024-11-16T16:15:28,127 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:28,127 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:28,182 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0640097e275b4afcb43e08062de112fb 2024-11-16T16:15:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/324a1bf77b454c3ab8e13a59037603e7 is 50, key is test_row_0/B:col10/1731773727287/Put/seqid=0 2024-11-16T16:15:28,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742151_1327 (size=12301) 2024-11-16T16:15:28,206 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/324a1bf77b454c3ab8e13a59037603e7 2024-11-16T16:15:28,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c86561f7a0634fe290a634ae3d7ddadc is 50, key is test_row_0/C:col10/1731773727287/Put/seqid=0 2024-11-16T16:15:28,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742152_1328 (size=12301) 2024-11-16T16:15:28,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773788275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773788275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773788585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773788585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:28,617 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c86561f7a0634fe290a634ae3d7ddadc 2024-11-16T16:15:28,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0640097e275b4afcb43e08062de112fb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb 2024-11-16T16:15:28,636 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T16:15:28,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/324a1bf77b454c3ab8e13a59037603e7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7 2024-11-16T16:15:28,642 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T16:15:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c86561f7a0634fe290a634ae3d7ddadc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc 2024-11-16T16:15:28,651 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc, entries=150, sequenceid=435, filesize=12.0 K 2024-11-16T16:15:28,652 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 1161dcb397ffa3f05e58f6711d26c4bb in 878ms, sequenceid=435, compaction requested=false 2024-11-16T16:15:28,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:28,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:28,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-16T16:15:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-16T16:15:28,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-16T16:15:28,659 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4950 sec 2024-11-16T16:15:28,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5050 sec 2024-11-16T16:15:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:29,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:29,100 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:29,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e2f9a7411de64a779412e5a4e5eb6c30 is 50, key is test_row_0/A:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:29,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742153_1329 (size=14741) 2024-11-16T16:15:29,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773789143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773789148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773789255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773789255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-16T16:15:29,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:29,305 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-16T16:15:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:29,307 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:29,308 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:29,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:29,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T16:15:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773789464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773789465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e2f9a7411de64a779412e5a4e5eb6c30 2024-11-16T16:15:29,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6e40679615de44229a8d30bcb4af22a3 is 50, key is test_row_0/B:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:29,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742154_1330 (size=12301) 2024-11-16T16:15:29,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6e40679615de44229a8d30bcb4af22a3 2024-11-16T16:15:29,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/662ab428b4f3427bb60824dedaf6c588 is 50, key is test_row_0/C:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:29,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742155_1331 (size=12301) 2024-11-16T16:15:29,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T16:15:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:29,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T16:15:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773789774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773789774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32772 deadline: 1731773789815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32816 deadline: 1731773789815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,819 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:29,819 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32818 deadline: 1731773789825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:29,828 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 1161dcb397ffa3f05e58f6711d26c4bb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:29,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:29,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T16:15:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:29,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/662ab428b4f3427bb60824dedaf6c588 2024-11-16T16:15:29,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/e2f9a7411de64a779412e5a4e5eb6c30 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30 2024-11-16T16:15:29,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30, entries=200, sequenceid=450, filesize=14.4 K 2024-11-16T16:15:29,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6e40679615de44229a8d30bcb4af22a3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3 2024-11-16T16:15:29,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3, entries=150, sequenceid=450, filesize=12.0 K 2024-11-16T16:15:29,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/662ab428b4f3427bb60824dedaf6c588 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588 2024-11-16T16:15:29,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588, entries=150, sequenceid=450, filesize=12.0 K 2024-11-16T16:15:29,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1161dcb397ffa3f05e58f6711d26c4bb in 895ms, sequenceid=450, compaction requested=true 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:29,993 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:29,993 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:29,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:29,994 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,994 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:29,994 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/95fc6851ae6c4e4fb371fb7b703d3a53, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.8 K 2024-11-16T16:15:29,994 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ca86b65cf0984a05ad3f285c23247ebf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=39.2 K 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 95fc6851ae6c4e4fb371fb7b703d3a53, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727127 2024-11-16T16:15:29,994 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca86b65cf0984a05ad3f285c23247ebf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727127 2024-11-16T16:15:29,995 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0640097e275b4afcb43e08062de112fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731773727280 2024-11-16T16:15:29,995 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 324a1bf77b454c3ab8e13a59037603e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731773727280 2024-11-16T16:15:29,995 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e40679615de44229a8d30bcb4af22a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:29,995 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f9a7411de64a779412e5a4e5eb6c30, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:30,001 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#287 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:30,001 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/7fc672fadb03455a8245f9224f267252 is 50, key is test_row_0/A:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:30,004 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:30,004 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7d6d0d0d2a65423bad4eb5f46e1ac150 is 50, key is test_row_0/B:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:30,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742156_1332 (size=13187) 2024-11-16T16:15:30,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742157_1333 (size=13187) 2024-11-16T16:15:30,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-16T16:15:30,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:30,072 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 is 50, key is test_row_0/A:col10/1731773729146/Put/seqid=0 2024-11-16T16:15:30,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742158_1334 (size=12301) 2024-11-16T16:15:30,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:30,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:30,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773790302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773790304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,409 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/7fc672fadb03455a8245f9224f267252 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7fc672fadb03455a8245f9224f267252 2024-11-16T16:15:30,415 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/7d6d0d0d2a65423bad4eb5f46e1ac150 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7d6d0d0d2a65423bad4eb5f46e1ac150 2024-11-16T16:15:30,415 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 7fc672fadb03455a8245f9224f267252(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:30,415 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:30,415 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773729993; duration=0sec 2024-11-16T16:15:30,415 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:30,415 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:30,415 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:30,416 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:30,416 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:30,416 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:30,416 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f76086613fd04414ba800c5ef85e0bd3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.8 K 2024-11-16T16:15:30,416 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f76086613fd04414ba800c5ef85e0bd3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731773727127 2024-11-16T16:15:30,417 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c86561f7a0634fe290a634ae3d7ddadc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=435, earliestPutTs=1731773727280 2024-11-16T16:15:30,417 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 662ab428b4f3427bb60824dedaf6c588, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773790415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773790415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,421 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 7d6d0d0d2a65423bad4eb5f46e1ac150(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:30,421 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:30,421 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773729993; duration=0sec 2024-11-16T16:15:30,421 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:30,421 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:30,425 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:30,425 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/543ee76847724b838c6f1d4428b45025 is 50, key is test_row_0/C:col10/1731773727935/Put/seqid=0 2024-11-16T16:15:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:30,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742159_1335 (size=13187) 2024-11-16T16:15:30,480 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 2024-11-16T16:15:30,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/0f8623e5605243268566ef9e812c90b8 is 50, key is test_row_0/B:col10/1731773729146/Put/seqid=0 2024-11-16T16:15:30,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742160_1336 (size=12301) 2024-11-16T16:15:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773790625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773790625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,861 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/543ee76847724b838c6f1d4428b45025 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/543ee76847724b838c6f1d4428b45025 2024-11-16T16:15:30,867 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 543ee76847724b838c6f1d4428b45025(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:30,868 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:30,868 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773729993; duration=0sec 2024-11-16T16:15:30,868 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:30,868 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:30,902 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/0f8623e5605243268566ef9e812c90b8 2024-11-16T16:15:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/517c8c9e27044d3d967dbcb39d2d9510 is 50, key is test_row_0/C:col10/1731773729146/Put/seqid=0 2024-11-16T16:15:30,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773790935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773790935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:30,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742161_1337 (size=12301) 2024-11-16T16:15:30,950 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/517c8c9e27044d3d967dbcb39d2d9510 2024-11-16T16:15:30,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 2024-11-16T16:15:30,961 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8, entries=150, sequenceid=470, filesize=12.0 K 2024-11-16T16:15:30,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/0f8623e5605243268566ef9e812c90b8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8 2024-11-16T16:15:30,967 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8, entries=150, sequenceid=470, filesize=12.0 K 2024-11-16T16:15:30,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/517c8c9e27044d3d967dbcb39d2d9510 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510 2024-11-16T16:15:30,973 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510, entries=150, sequenceid=470, filesize=12.0 K 2024-11-16T16:15:30,974 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1161dcb397ffa3f05e58f6711d26c4bb in 902ms, sequenceid=470, compaction requested=false 2024-11-16T16:15:30,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:30,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:30,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-16T16:15:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-16T16:15:30,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-16T16:15:30,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6680 sec 2024-11-16T16:15:30,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6740 sec 2024-11-16T16:15:31,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:31,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:31,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:31,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/f5d2b328cffe4091943dc0b6a5c8ce61 is 50, key is test_row_0/A:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-16T16:15:31,455 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:31,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:31,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-16T16:15:31,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742162_1338 (size=12301) 2024-11-16T16:15:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T16:15:31,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:31,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:31,459 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:31,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/f5d2b328cffe4091943dc0b6a5c8ce61 2024-11-16T16:15:31,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/42006d5e69f3465982342eaee80a5bb0 is 50, key is test_row_0/B:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:31,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742163_1339 (size=12301) 2024-11-16T16:15:31,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/42006d5e69f3465982342eaee80a5bb0 2024-11-16T16:15:31,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b83b747c331140769a31723dded4f343 is 50, key is test_row_0/C:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:31,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773791493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773791494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742164_1340 (size=12301) 2024-11-16T16:15:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T16:15:31,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773791605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773791605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T16:15:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T16:15:31,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:31,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T16:15:31,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773791815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773791815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:31,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:31,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=486 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b83b747c331140769a31723dded4f343 2024-11-16T16:15:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/f5d2b328cffe4091943dc0b6a5c8ce61 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61 2024-11-16T16:15:31,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61, entries=150, sequenceid=486, filesize=12.0 K 2024-11-16T16:15:31,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T16:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:31,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/42006d5e69f3465982342eaee80a5bb0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0 2024-11-16T16:15:31,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0, entries=150, sequenceid=486, filesize=12.0 K 2024-11-16T16:15:31,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/b83b747c331140769a31723dded4f343 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343 2024-11-16T16:15:31,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343, entries=150, sequenceid=486, filesize=12.0 K 2024-11-16T16:15:31,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1161dcb397ffa3f05e58f6711d26c4bb in 485ms, sequenceid=486, compaction requested=true 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:31,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:31,932 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:31,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:31,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:31,935 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,935 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,935 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7d6d0d0d2a65423bad4eb5f46e1ac150, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.9 K 2024-11-16T16:15:31,935 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7fc672fadb03455a8245f9224f267252, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.9 K 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc672fadb03455a8245f9224f267252, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:31,935 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d6d0d0d2a65423bad4eb5f46e1ac150, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:31,936 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad7d64050ea46d3b6ac04a6c2cfd2b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731773729134 2024-11-16T16:15:31,936 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8623e5605243268566ef9e812c90b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731773729134 2024-11-16T16:15:31,936 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d2b328cffe4091943dc0b6a5c8ce61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:31,936 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 42006d5e69f3465982342eaee80a5bb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:31,944 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:31,945 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/7bcf8c8744ba4f1a94429774f0c5fbaf is 50, key is test_row_0/A:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:31,947 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:31,948 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/f430a7de5e924bfcbd3517a8ea733d5b is 50, key is test_row_0/B:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:31,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742165_1341 (size=13289) 2024-11-16T16:15:31,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742166_1342 (size=13289) 2024-11-16T16:15:31,965 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/7bcf8c8744ba4f1a94429774f0c5fbaf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7bcf8c8744ba4f1a94429774f0c5fbaf 2024-11-16T16:15:31,968 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/f430a7de5e924bfcbd3517a8ea733d5b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f430a7de5e924bfcbd3517a8ea733d5b 2024-11-16T16:15:31,976 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 7bcf8c8744ba4f1a94429774f0c5fbaf(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:31,976 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into f430a7de5e924bfcbd3517a8ea733d5b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:31,976 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=13, startTime=1731773731932; duration=0sec 2024-11-16T16:15:31,976 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=13, startTime=1731773731932; duration=0sec 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:31,976 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:31,978 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:31,978 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:31,978 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:31,978 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/543ee76847724b838c6f1d4428b45025, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=36.9 K 2024-11-16T16:15:31,978 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 543ee76847724b838c6f1d4428b45025, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731773727935 2024-11-16T16:15:31,978 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 517c8c9e27044d3d967dbcb39d2d9510, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731773729134 2024-11-16T16:15:31,979 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b83b747c331140769a31723dded4f343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:31,991 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:31,992 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c0bc2c853a644f2791a6dcb25852d2a7 is 50, key is test_row_0/C:col10/1731773731446/Put/seqid=0 2024-11-16T16:15:32,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742167_1343 (size=13289) 2024-11-16T16:15:32,007 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c0bc2c853a644f2791a6dcb25852d2a7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c0bc2c853a644f2791a6dcb25852d2a7 2024-11-16T16:15:32,011 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into c0bc2c853a644f2791a6dcb25852d2a7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:32,011 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:32,011 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=13, startTime=1731773731932; duration=0sec 2024-11-16T16:15:32,011 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:32,011 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:32,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-16T16:15:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:32,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/db13fe23a19c41aca0ace34ec16b79aa is 50, key is test_row_0/A:col10/1731773731484/Put/seqid=0 2024-11-16T16:15:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T16:15:32,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742168_1344 (size=12301) 2024-11-16T16:15:32,093 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/db13fe23a19c41aca0ace34ec16b79aa 2024-11-16T16:15:32,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c993c47535cc4a2db852194076adafaa is 50, key is test_row_0/B:col10/1731773731484/Put/seqid=0 2024-11-16T16:15:32,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742169_1345 (size=12301) 2024-11-16T16:15:32,103 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c993c47535cc4a2db852194076adafaa 2024-11-16T16:15:32,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/95a3151c906641eb82cb0fdf50fd9b3c is 50, key is test_row_0/C:col10/1731773731484/Put/seqid=0 2024-11-16T16:15:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742170_1346 (size=12301) 2024-11-16T16:15:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:32,138 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:32,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773792155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773792159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773792265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773792265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773792475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773792475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,513 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/95a3151c906641eb82cb0fdf50fd9b3c 2024-11-16T16:15:32,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/db13fe23a19c41aca0ace34ec16b79aa as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa 2024-11-16T16:15:32,520 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T16:15:32,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/c993c47535cc4a2db852194076adafaa as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa 2024-11-16T16:15:32,528 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T16:15:32,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/95a3151c906641eb82cb0fdf50fd9b3c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c 2024-11-16T16:15:32,533 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c, entries=150, sequenceid=508, filesize=12.0 K 2024-11-16T16:15:32,533 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1161dcb397ffa3f05e58f6711d26c4bb in 459ms, sequenceid=508, compaction requested=false 2024-11-16T16:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-16T16:15:32,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-16T16:15:32,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-16T16:15:32,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-16T16:15:32,538 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.0810 sec 2024-11-16T16:15:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-16T16:15:32,595 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:32,596 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-16T16:15:32,597 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T16:15:32,598 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:32,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T16:15:32,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-16T16:15:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:32,750 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:32,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/4dedc55dfc504378aa5bc8ecbf730508 is 50, key is test_row_0/A:col10/1731773732154/Put/seqid=0 2024-11-16T16:15:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742171_1347 (size=12301) 2024-11-16T16:15:32,758 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/4dedc55dfc504378aa5bc8ecbf730508 2024-11-16T16:15:32,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cc743b6394474d59b2497ef7a118c4de is 50, key is test_row_0/B:col10/1731773732154/Put/seqid=0 2024-11-16T16:15:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742172_1348 (size=12301) 2024-11-16T16:15:32,769 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cc743b6394474d59b2497ef7a118c4de 2024-11-16T16:15:32,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d025f868551644fe857e7a9665a01733 is 50, key is test_row_0/C:col10/1731773732154/Put/seqid=0 2024-11-16T16:15:32,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:32,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:32,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742173_1349 (size=12301) 2024-11-16T16:15:32,802 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d025f868551644fe857e7a9665a01733 2024-11-16T16:15:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/4dedc55dfc504378aa5bc8ecbf730508 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508 2024-11-16T16:15:32,812 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508, entries=150, sequenceid=521, filesize=12.0 K 2024-11-16T16:15:32,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cc743b6394474d59b2497ef7a118c4de as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de 2024-11-16T16:15:32,817 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de, entries=150, sequenceid=521, filesize=12.0 K 2024-11-16T16:15:32,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/d025f868551644fe857e7a9665a01733 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733 2024-11-16T16:15:32,822 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733, entries=150, sequenceid=521, filesize=12.0 K 2024-11-16T16:15:32,822 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=67.09 KB/68700 for 1161dcb397ffa3f05e58f6711d26c4bb in 72ms, sequenceid=521, compaction requested=true 2024-11-16T16:15:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-16T16:15:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:32,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:32,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:32,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-16T16:15:32,828 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 226 msec 2024-11-16T16:15:32,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b74cdb737c94ee1b8b45a702e283361 is 50, key is test_row_0/A:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:32,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 237 msec 2024-11-16T16:15:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742174_1350 (size=14741) 2024-11-16T16:15:32,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-16T16:15:32,915 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:32,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees 2024-11-16T16:15:32,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:32,968 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:32,968 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=86, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:32,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773792975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773792975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:32,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773793085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773793085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b74cdb737c94ee1b8b45a702e283361 2024-11-16T16:15:33,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cd24791333a54a5bb8b435c295354d16 is 50, key is test_row_0/B:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:33,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742175_1351 (size=12301) 2024-11-16T16:15:33,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cd24791333a54a5bb8b435c295354d16 2024-11-16T16:15:33,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/9361adc9c7dd4c4bb5e983580ca67273 is 50, key is test_row_0/C:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:33,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742176_1352 (size=12301) 2024-11-16T16:15:33,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:33,285 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/9361adc9c7dd4c4bb5e983580ca67273 2024-11-16T16:15:33,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b74cdb737c94ee1b8b45a702e283361 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361 2024-11-16T16:15:33,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361, entries=200, sequenceid=535, filesize=14.4 K 2024-11-16T16:15:33,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/cd24791333a54a5bb8b435c295354d16 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16 2024-11-16T16:15:33,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16, entries=150, sequenceid=535, filesize=12.0 K 2024-11-16T16:15:33,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/9361adc9c7dd4c4bb5e983580ca67273 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273 2024-11-16T16:15:33,302 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273, entries=150, sequenceid=535, filesize=12.0 K 2024-11-16T16:15:33,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1161dcb397ffa3f05e58f6711d26c4bb in 480ms, sequenceid=535, compaction requested=true 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:33,304 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:33,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-16T16:15:33,304 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:33,304 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:33,305 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:33,307 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:33,307 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:33,307 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:33,307 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:33,307 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,308 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,308 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f430a7de5e924bfcbd3517a8ea733d5b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=49.0 K 2024-11-16T16:15:33,308 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7bcf8c8744ba4f1a94429774f0c5fbaf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=51.4 K 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f430a7de5e924bfcbd3517a8ea733d5b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bcf8c8744ba4f1a94429774f0c5fbaf, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c993c47535cc4a2db852194076adafaa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731773731478 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting db13fe23a19c41aca0ace34ec16b79aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731773731478 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cc743b6394474d59b2497ef7a118c4de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731773732153 2024-11-16T16:15:33,308 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dedc55dfc504378aa5bc8ecbf730508, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731773732153 2024-11-16T16:15:33,309 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cd24791333a54a5bb8b435c295354d16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732815 2024-11-16T16:15:33,309 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b74cdb737c94ee1b8b45a702e283361, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732794 2024-11-16T16:15:33,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/937a2bf639d04cf69fcd49bf3a971058 is 50, key is test_row_0/A:col10/1731773732974/Put/seqid=0 2024-11-16T16:15:33,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742177_1353 (size=14741) 2024-11-16T16:15:33,323 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:33,323 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/5fa7225d5f7d44db96658b232ef68267 is 50, key is test_row_0/B:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:33,327 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:33,327 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/1d2e3a7241c14dc1babbbbf33ffbf6f9 is 50, key is test_row_0/A:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:33,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773793335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773793337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742178_1354 (size=13425) 2024-11-16T16:15:33,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742179_1355 (size=13425) 2024-11-16T16:15:33,359 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/1d2e3a7241c14dc1babbbbf33ffbf6f9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1d2e3a7241c14dc1babbbbf33ffbf6f9 2024-11-16T16:15:33,360 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/5fa7225d5f7d44db96658b232ef68267 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5fa7225d5f7d44db96658b232ef68267 2024-11-16T16:15:33,367 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 1d2e3a7241c14dc1babbbbf33ffbf6f9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:33,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:33,367 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=12, startTime=1731773733304; duration=0sec 2024-11-16T16:15:33,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:33,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:33,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:33,369 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:33,369 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:33,369 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,369 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c0bc2c853a644f2791a6dcb25852d2a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=49.0 K 2024-11-16T16:15:33,370 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 5fa7225d5f7d44db96658b232ef68267(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0bc2c853a644f2791a6dcb25852d2a7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=486, earliestPutTs=1731773730290 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:33,370 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=12, startTime=1731773733304; duration=0sec 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95a3151c906641eb82cb0fdf50fd9b3c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731773731478 2024-11-16T16:15:33,370 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d025f868551644fe857e7a9665a01733, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731773732153 2024-11-16T16:15:33,371 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9361adc9c7dd4c4bb5e983580ca67273, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732815 2024-11-16T16:15:33,379 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#311 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:33,379 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/2178642377f0479091eac4a2c637d12f is 50, key is test_row_0/C:col10/1731773732815/Put/seqid=0 2024-11-16T16:15:33,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742180_1356 (size=13425) 2024-11-16T16:15:33,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773793444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773793445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:33,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773793655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773793655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/937a2bf639d04cf69fcd49bf3a971058 2024-11-16T16:15:33,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/f6802827c9ec4b3eb956b47922b12e23 is 50, key is test_row_0/B:col10/1731773732974/Put/seqid=0 2024-11-16T16:15:33,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742181_1357 (size=12301) 2024-11-16T16:15:33,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/f6802827c9ec4b3eb956b47922b12e23 2024-11-16T16:15:33,806 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/2178642377f0479091eac4a2c637d12f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/2178642377f0479091eac4a2c637d12f 2024-11-16T16:15:33,817 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into 2178642377f0479091eac4a2c637d12f(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:33,817 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:33,817 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=12, startTime=1731773733304; duration=0sec 2024-11-16T16:15:33,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/bcf15a221f1747009a24b4c93bdec694 is 50, key is test_row_0/C:col10/1731773732974/Put/seqid=0 2024-11-16T16:15:33,817 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:33,817 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742182_1358 (size=12301) 2024-11-16T16:15:33,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:33,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:33,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:33,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:33,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773793965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773793965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:33,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:34,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:34,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:34,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:34,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] handler.RSProcedureHandler(58): pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=87 java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=87 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:34,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/bcf15a221f1747009a24b4c93bdec694 2024-11-16T16:15:34,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/937a2bf639d04cf69fcd49bf3a971058 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058 2024-11-16T16:15:34,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058, entries=200, sequenceid=555, filesize=14.4 K 2024-11-16T16:15:34,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/f6802827c9ec4b3eb956b47922b12e23 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23 2024-11-16T16:15:34,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23, entries=150, sequenceid=555, filesize=12.0 K 2024-11-16T16:15:34,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/bcf15a221f1747009a24b4c93bdec694 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694 2024-11-16T16:15:34,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694, entries=150, sequenceid=555, filesize=12.0 K 2024-11-16T16:15:34,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 1161dcb397ffa3f05e58f6711d26c4bb in 968ms, sequenceid=555, compaction requested=false 2024-11-16T16:15:34,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:34,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=87 2024-11-16T16:15:34,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:34,354 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:34,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/17c0bb8f6ee44d54bb5f98a580f414fd is 50, key is test_row_0/A:col10/1731773733335/Put/seqid=0 2024-11-16T16:15:34,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742183_1359 (size=9857) 2024-11-16T16:15:34,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. as already flushing 2024-11-16T16:15:34,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773794511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773794512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773794625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773794625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,762 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/17c0bb8f6ee44d54bb5f98a580f414fd 2024-11-16T16:15:34,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/40c987e067cc44e381a52427ac0b9464 is 50, key is test_row_0/B:col10/1731773733335/Put/seqid=0 2024-11-16T16:15:34,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742184_1360 (size=9857) 2024-11-16T16:15:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773794834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773794835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:34,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60990 deadline: 1731773795145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:35,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:35,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:35,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:32820 deadline: 1731773795146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:35,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:15:35,172 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/40c987e067cc44e381a52427ac0b9464 2024-11-16T16:15:35,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/288b7c84c5db423fa28f06fdf5a8f578 is 50, key is test_row_0/C:col10/1731773733335/Put/seqid=0 2024-11-16T16:15:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742185_1361 (size=9857) 2024-11-16T16:15:35,273 INFO [Thread-1182 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1182. 2024-11-16T16:15:35,273 DEBUG [Thread-1182 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,273 INFO [Thread-1180 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1180. 2024-11-16T16:15:35,273 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,273 DEBUG [Thread-1180 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,273 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,273 DEBUG [Thread-1182 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,273 DEBUG [Thread-1180 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,274 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,275 INFO [Thread-1186 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1186. 2024-11-16T16:15:35,276 DEBUG [Thread-1186 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,276 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,276 DEBUG [Thread-1186 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,276 INFO [Thread-1188 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1188. 2024-11-16T16:15:35,276 DEBUG [Thread-1188 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,276 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,276 DEBUG [Thread-1188 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,279 INFO [Thread-1184 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1184. 2024-11-16T16:15:35,279 DEBUG [Thread-1184 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,279 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,279 DEBUG [Thread-1184 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,279 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,588 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=570 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/288b7c84c5db423fa28f06fdf5a8f578 2024-11-16T16:15:35,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/17c0bb8f6ee44d54bb5f98a580f414fd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd 2024-11-16T16:15:35,602 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd, entries=100, sequenceid=570, filesize=9.6 K 2024-11-16T16:15:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/40c987e067cc44e381a52427ac0b9464 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464 2024-11-16T16:15:35,606 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464, entries=100, sequenceid=570, filesize=9.6 K 2024-11-16T16:15:35,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/288b7c84c5db423fa28f06fdf5a8f578 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578 2024-11-16T16:15:35,610 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578, entries=100, sequenceid=570, filesize=9.6 K 2024-11-16T16:15:35,611 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 1161dcb397ffa3f05e58f6711d26c4bb in 1257ms, sequenceid=570, compaction requested=true 2024-11-16T16:15:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=87}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=87 2024-11-16T16:15:35,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=87 2024-11-16T16:15:35,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-16T16:15:35,614 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6450 sec 2024-11-16T16:15:35,616 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=86, table=TestAcidGuarantees in 2.6990 sec 2024-11-16T16:15:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:35,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-16T16:15:35,656 INFO [Thread-1175 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1175. 2024-11-16T16:15:35,657 DEBUG [Thread-1175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,657 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,657 INFO [Thread-1169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1169. 2024-11-16T16:15:35,657 DEBUG [Thread-1175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:35,657 DEBUG [Thread-1169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:35,657 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:35,657 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:35,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:35,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:35,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:35,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a468260150034036bc072c35713f52e1 is 50, key is test_row_0/A:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742186_1362 (size=12301) 2024-11-16T16:15:36,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a468260150034036bc072c35713f52e1 2024-11-16T16:15:36,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/a253513335c84b21a455bdad3ad82813 is 50, key is test_row_0/B:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:36,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742187_1363 (size=12301) 2024-11-16T16:15:36,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/a253513335c84b21a455bdad3ad82813 2024-11-16T16:15:36,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/5d151fce36204f86aeca58e7526346dc is 50, key is test_row_0/C:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:36,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742188_1364 (size=12301) 2024-11-16T16:15:36,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=592 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/5d151fce36204f86aeca58e7526346dc 2024-11-16T16:15:36,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/a468260150034036bc072c35713f52e1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1 2024-11-16T16:15:36,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1, entries=150, sequenceid=592, filesize=12.0 K 2024-11-16T16:15:36,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/a253513335c84b21a455bdad3ad82813 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813 2024-11-16T16:15:36,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813, entries=150, sequenceid=592, filesize=12.0 K 2024-11-16T16:15:36,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/5d151fce36204f86aeca58e7526346dc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc 2024-11-16T16:15:36,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc, entries=150, sequenceid=592, filesize=12.0 K 2024-11-16T16:15:36,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=0 B/0 for 1161dcb397ffa3f05e58f6711d26c4bb in 1253ms, sequenceid=592, compaction requested=true 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:36,909 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:36,909 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1161dcb397ffa3f05e58f6711d26c4bb:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:36,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:36,910 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47884 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:36,910 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50324 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:36,910 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/A is initiating minor compaction (all files) 2024-11-16T16:15:36,910 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/B is initiating minor compaction (all files) 2024-11-16T16:15:36,911 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/A in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:36,911 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/B in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:36,911 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1d2e3a7241c14dc1babbbbf33ffbf6f9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=49.1 K 2024-11-16T16:15:36,911 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5fa7225d5f7d44db96658b232ef68267, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=46.8 K 2024-11-16T16:15:36,911 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d2e3a7241c14dc1babbbbf33ffbf6f9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732815 2024-11-16T16:15:36,911 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fa7225d5f7d44db96658b232ef68267, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732815 2024-11-16T16:15:36,911 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f6802827c9ec4b3eb956b47922b12e23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731773732922 2024-11-16T16:15:36,911 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 937a2bf639d04cf69fcd49bf3a971058, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731773732902 2024-11-16T16:15:36,912 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 40c987e067cc44e381a52427ac0b9464, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731773733326 2024-11-16T16:15:36,912 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17c0bb8f6ee44d54bb5f98a580f414fd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731773733326 2024-11-16T16:15:36,912 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a253513335c84b21a455bdad3ad82813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731773734506 2024-11-16T16:15:36,912 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a468260150034036bc072c35713f52e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731773734506 2024-11-16T16:15:36,920 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#B#compaction#320 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:36,921 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/74bfbb327ed34dc88fb39090f962b9e5 is 50, key is test_row_0/B:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:36,921 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#A#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:36,922 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b43177be9414793bbdf1f12cfd4a5bd is 50, key is test_row_0/A:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:36,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742189_1365 (size=13561) 2024-11-16T16:15:36,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742190_1366 (size=13561) 2024-11-16T16:15:36,928 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/8b43177be9414793bbdf1f12cfd4a5bd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b43177be9414793bbdf1f12cfd4a5bd 2024-11-16T16:15:36,931 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/A of 1161dcb397ffa3f05e58f6711d26c4bb into 8b43177be9414793bbdf1f12cfd4a5bd(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:36,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:36,932 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/A, priority=12, startTime=1731773736909; duration=0sec 2024-11-16T16:15:36,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:36,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:A 2024-11-16T16:15:36,932 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:36,933 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47884 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:36,933 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 1161dcb397ffa3f05e58f6711d26c4bb/C is initiating minor compaction (all files) 2024-11-16T16:15:36,933 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1161dcb397ffa3f05e58f6711d26c4bb/C in TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:36,933 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/2178642377f0479091eac4a2c637d12f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp, totalSize=46.8 K 2024-11-16T16:15:36,933 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2178642377f0479091eac4a2c637d12f, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731773732815 2024-11-16T16:15:36,933 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcf15a221f1747009a24b4c93bdec694, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1731773732922 2024-11-16T16:15:36,933 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 288b7c84c5db423fa28f06fdf5a8f578, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=570, earliestPutTs=1731773733326 2024-11-16T16:15:36,934 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d151fce36204f86aeca58e7526346dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=592, earliestPutTs=1731773734506 2024-11-16T16:15:36,940 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1161dcb397ffa3f05e58f6711d26c4bb#C#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:36,940 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c223f8a596b1434896278537e91bda73 is 50, key is test_row_0/C:col10/1731773734506/Put/seqid=0 2024-11-16T16:15:36,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742191_1367 (size=13561) 2024-11-16T16:15:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-16T16:15:37,136 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:37,334 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/74bfbb327ed34dc88fb39090f962b9e5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/74bfbb327ed34dc88fb39090f962b9e5 2024-11-16T16:15:37,342 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/B of 1161dcb397ffa3f05e58f6711d26c4bb into 74bfbb327ed34dc88fb39090f962b9e5(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:37,342 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:37,342 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/B, priority=12, startTime=1731773736909; duration=0sec 2024-11-16T16:15:37,342 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:37,342 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:B 2024-11-16T16:15:37,348 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/c223f8a596b1434896278537e91bda73 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c223f8a596b1434896278537e91bda73 2024-11-16T16:15:37,352 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1161dcb397ffa3f05e58f6711d26c4bb/C of 1161dcb397ffa3f05e58f6711d26c4bb into c223f8a596b1434896278537e91bda73(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:37,352 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1161dcb397ffa3f05e58f6711d26c4bb: 2024-11-16T16:15:37,352 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., storeName=1161dcb397ffa3f05e58f6711d26c4bb/C, priority=12, startTime=1731773736909; duration=0sec 2024-11-16T16:15:37,353 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:37,353 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1161dcb397ffa3f05e58f6711d26c4bb:C 2024-11-16T16:15:39,846 INFO [Thread-1173 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1173. 2024-11-16T16:15:39,846 DEBUG [Thread-1173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:39,846 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,847 DEBUG [Thread-1173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:39,876 INFO [Thread-1171 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1171. 2024-11-16T16:15:39,876 DEBUG [Thread-1171 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:39,876 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,876 DEBUG [Thread-1171 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,876 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:39,896 INFO [Thread-1177 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1177. 2024-11-16T16:15:39,896 DEBUG [Thread-1177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:15:39,896 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,897 DEBUG [Thread-1177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 148 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3013 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3121 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2950 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3020 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3104 2024-11-16T16:15:39,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:15:39,897 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:15:39,897 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:15:39,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:39,898 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:15:39,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:15:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:39,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773739902"}]},"ts":"1731773739902"} 2024-11-16T16:15:39,905 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:15:39,905 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:15:39,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:15:39,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, UNASSIGN}] 2024-11-16T16:15:39,910 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, UNASSIGN 2024-11-16T16:15:39,911 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1161dcb397ffa3f05e58f6711d26c4bb, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:39,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, UNASSIGN because future has completed 2024-11-16T16:15:39,913 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:15:39,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:40,067 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(122): Close 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:40,067 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:15:40,068 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1722): Closing 1161dcb397ffa3f05e58f6711d26c4bb, disabling compactions & flushes 2024-11-16T16:15:40,068 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:40,068 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:40,068 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. after waiting 0 ms 2024-11-16T16:15:40,068 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:40,068 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(2902): Flushing 1161dcb397ffa3f05e58f6711d26c4bb 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=A 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=B 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1161dcb397ffa3f05e58f6711d26c4bb, store=C 2024-11-16T16:15:40,069 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:40,076 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/48a831504559491ba0355e73c2a68d7c is 50, key is test_row_0/A:col10/1731773739845/Put/seqid=0 2024-11-16T16:15:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742192_1368 (size=9857) 2024-11-16T16:15:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:40,480 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/48a831504559491ba0355e73c2a68d7c 2024-11-16T16:15:40,492 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6a3c7824634c4ba48e5d326afdc0d354 is 50, key is test_row_0/B:col10/1731773739845/Put/seqid=0 2024-11-16T16:15:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742193_1369 (size=9857) 2024-11-16T16:15:40,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:40,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:15:40,897 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6a3c7824634c4ba48e5d326afdc0d354 2024-11-16T16:15:40,904 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/31b1b56fc32d414a80bea03a679e3e34 is 50, key is test_row_0/C:col10/1731773739845/Put/seqid=0 2024-11-16T16:15:40,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742194_1370 (size=9857) 2024-11-16T16:15:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:41,309 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/31b1b56fc32d414a80bea03a679e3e34 2024-11-16T16:15:41,312 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/A/48a831504559491ba0355e73c2a68d7c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/48a831504559491ba0355e73c2a68d7c 2024-11-16T16:15:41,316 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/48a831504559491ba0355e73c2a68d7c, entries=100, sequenceid=601, filesize=9.6 K 2024-11-16T16:15:41,317 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/B/6a3c7824634c4ba48e5d326afdc0d354 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6a3c7824634c4ba48e5d326afdc0d354 2024-11-16T16:15:41,320 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6a3c7824634c4ba48e5d326afdc0d354, entries=100, sequenceid=601, filesize=9.6 K 2024-11-16T16:15:41,321 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/.tmp/C/31b1b56fc32d414a80bea03a679e3e34 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/31b1b56fc32d414a80bea03a679e3e34 2024-11-16T16:15:41,324 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/31b1b56fc32d414a80bea03a679e3e34, entries=100, sequenceid=601, filesize=9.6 K 2024-11-16T16:15:41,325 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1161dcb397ffa3f05e58f6711d26c4bb in 1257ms, sequenceid=601, compaction requested=false 2024-11-16T16:15:41,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e3cbde42592048c09c24d8f1debe05ea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0688bac533494608afb2fae672a38bc0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/acf44847a0464730b77cc0936bc35b9e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6b0d008713e14d8a8bd8aa7beb8d9ff0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8d73e29b1013454c9694bf723c694edb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/470cbc7e40504d3d8aec97de2599b439, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ca86b65cf0984a05ad3f285c23247ebf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7fc672fadb03455a8245f9224f267252, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7bcf8c8744ba4f1a94429774f0c5fbaf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1d2e3a7241c14dc1babbbbf33ffbf6f9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1] to archive 2024-11-16T16:15:41,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:41,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ae26508ba8c4479ea93e3f974f5d1bbf 2024-11-16T16:15:41,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ec32546efad041b99003e08db4dd9b1d 2024-11-16T16:15:41,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6608ebc62b054828820008eacdc6e958 2024-11-16T16:15:41,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e3cbde42592048c09c24d8f1debe05ea to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e3cbde42592048c09c24d8f1debe05ea 2024-11-16T16:15:41,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/19c3b087d5804136a650640bd2f0a6ce 2024-11-16T16:15:41,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0688bac533494608afb2fae672a38bc0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0688bac533494608afb2fae672a38bc0 2024-11-16T16:15:41,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e094b5762bae4b9987dfbf408666af1c 2024-11-16T16:15:41,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a3180ee649e645759435f1063c2d44bf 2024-11-16T16:15:41,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/2b8de78f272b494ea6e7a0757214ef2b 2024-11-16T16:15:41,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/acf44847a0464730b77cc0936bc35b9e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/acf44847a0464730b77cc0936bc35b9e 2024-11-16T16:15:41,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8123e36e754c4a0488edc6884e07fb6b 2024-11-16T16:15:41,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/013d3bf35bb64daeb1c23dc74ae3b7b3 2024-11-16T16:15:41,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6b0d008713e14d8a8bd8aa7beb8d9ff0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/6b0d008713e14d8a8bd8aa7beb8d9ff0 2024-11-16T16:15:41,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1286e7f4f7c747f7a17c6ad1d6f98247 2024-11-16T16:15:41,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b139e36e3faf4b1b8cb4c802bce9e95e 2024-11-16T16:15:41,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8701178717f84df49ea4f5ef43d230cc 2024-11-16T16:15:41,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/11191a59b7464f6889eb65741bccc573 2024-11-16T16:15:41,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8d73e29b1013454c9694bf723c694edb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8d73e29b1013454c9694bf723c694edb 2024-11-16T16:15:41,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/70c22e6d33e14f489e81e14ff4738cb3 2024-11-16T16:15:41,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b7ce42302e74ebf9054b14035cf09f8 2024-11-16T16:15:41,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/470cbc7e40504d3d8aec97de2599b439 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/470cbc7e40504d3d8aec97de2599b439 2024-11-16T16:15:41,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0d24fc2c61734f78aef0cbc32d1dcc71 2024-11-16T16:15:41,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/b446dac100fd4c95894e880bd3d22496 2024-11-16T16:15:41,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ca86b65cf0984a05ad3f285c23247ebf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/ca86b65cf0984a05ad3f285c23247ebf 2024-11-16T16:15:41,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0640097e275b4afcb43e08062de112fb 2024-11-16T16:15:41,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/e2f9a7411de64a779412e5a4e5eb6c30 2024-11-16T16:15:41,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7fc672fadb03455a8245f9224f267252 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7fc672fadb03455a8245f9224f267252 2024-11-16T16:15:41,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/0ad7d64050ea46d3b6ac04a6c2cfd2b8 2024-11-16T16:15:41,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7bcf8c8744ba4f1a94429774f0c5fbaf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/7bcf8c8744ba4f1a94429774f0c5fbaf 2024-11-16T16:15:41,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/f5d2b328cffe4091943dc0b6a5c8ce61 2024-11-16T16:15:41,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/db13fe23a19c41aca0ace34ec16b79aa 2024-11-16T16:15:41,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/4dedc55dfc504378aa5bc8ecbf730508 2024-11-16T16:15:41,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b74cdb737c94ee1b8b45a702e283361 2024-11-16T16:15:41,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1d2e3a7241c14dc1babbbbf33ffbf6f9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/1d2e3a7241c14dc1babbbbf33ffbf6f9 2024-11-16T16:15:41,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/937a2bf639d04cf69fcd49bf3a971058 2024-11-16T16:15:41,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/17c0bb8f6ee44d54bb5f98a580f414fd 2024-11-16T16:15:41,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/a468260150034036bc072c35713f52e1 2024-11-16T16:15:41,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/31484bf9b3c341adbb0265c23bdea0b9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/bbbdace1ac044d1d87650f1aef8c2d19, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/efaab7eb026d4179b0a72ec1829fd25d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/77d24687645c4cfd9d113b36e44f7f1c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/529b57bf376548cea7e591c75cd0e04b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7706913c6a034913b229ff9ffdd3814f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/95fc6851ae6c4e4fb371fb7b703d3a53, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7d6d0d0d2a65423bad4eb5f46e1ac150, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f430a7de5e924bfcbd3517a8ea733d5b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5fa7225d5f7d44db96658b232ef68267, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813] to archive 2024-11-16T16:15:41,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:41,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/9890c63c69b44cc09b653685fdf55419 2024-11-16T16:15:41,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/03e30b2d36c644318d13c846ea01a0d5 2024-11-16T16:15:41,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/31484bf9b3c341adbb0265c23bdea0b9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/31484bf9b3c341adbb0265c23bdea0b9 2024-11-16T16:15:41,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/01334e4a0e504315862855eafa8dd9da 2024-11-16T16:15:41,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3e0a7c7a68904e5399a5e7eaa32a4071 2024-11-16T16:15:41,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/bbbdace1ac044d1d87650f1aef8c2d19 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/bbbdace1ac044d1d87650f1aef8c2d19 2024-11-16T16:15:41,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/72422285e3e545d195a1cc7cd51d1d4b 2024-11-16T16:15:41,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1106df3cf17d45709c4e317f3a6fb857 2024-11-16T16:15:41,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/b006c3e85e434e0a88aea5bf11e2321d 2024-11-16T16:15:41,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/efaab7eb026d4179b0a72ec1829fd25d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/efaab7eb026d4179b0a72ec1829fd25d 2024-11-16T16:15:41,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/3ecd732fb29a4aff9a5764e5015cde14 2024-11-16T16:15:41,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/ac134cb35e884da59188384cfd0d6125 2024-11-16T16:15:41,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/77d24687645c4cfd9d113b36e44f7f1c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/77d24687645c4cfd9d113b36e44f7f1c 2024-11-16T16:15:41,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/1ce8d49a56a9411cac5232668fd62a07 2024-11-16T16:15:41,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c27e4650b2a14907878b61f5738b8e95 2024-11-16T16:15:41,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5b72e1e968d94614a33dad1705644680 2024-11-16T16:15:41,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/529b57bf376548cea7e591c75cd0e04b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/529b57bf376548cea7e591c75cd0e04b 2024-11-16T16:15:41,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87c75a57362b4b1194b58c9459db585b 2024-11-16T16:15:41,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7e139999b8b049f1802ea654c5a3f68c 2024-11-16T16:15:41,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7706913c6a034913b229ff9ffdd3814f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7706913c6a034913b229ff9ffdd3814f 2024-11-16T16:15:41,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/fdf8f3ffb07a4dae91e6ce53f05599ea 2024-11-16T16:15:41,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/813978114b034923b645f529896648d4 2024-11-16T16:15:41,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/95fc6851ae6c4e4fb371fb7b703d3a53 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/95fc6851ae6c4e4fb371fb7b703d3a53 2024-11-16T16:15:41,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/87fc5c3d4d7f45b986bf712074f28b69 2024-11-16T16:15:41,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/324a1bf77b454c3ab8e13a59037603e7 2024-11-16T16:15:41,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7d6d0d0d2a65423bad4eb5f46e1ac150 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/7d6d0d0d2a65423bad4eb5f46e1ac150 2024-11-16T16:15:41,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6e40679615de44229a8d30bcb4af22a3 2024-11-16T16:15:41,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/0f8623e5605243268566ef9e812c90b8 2024-11-16T16:15:41,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f430a7de5e924bfcbd3517a8ea733d5b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f430a7de5e924bfcbd3517a8ea733d5b 2024-11-16T16:15:41,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/42006d5e69f3465982342eaee80a5bb0 2024-11-16T16:15:41,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/c993c47535cc4a2db852194076adafaa 2024-11-16T16:15:41,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cc743b6394474d59b2497ef7a118c4de 2024-11-16T16:15:41,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5fa7225d5f7d44db96658b232ef68267 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/5fa7225d5f7d44db96658b232ef68267 2024-11-16T16:15:41,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/cd24791333a54a5bb8b435c295354d16 2024-11-16T16:15:41,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/f6802827c9ec4b3eb956b47922b12e23 2024-11-16T16:15:41,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/40c987e067cc44e381a52427ac0b9464 2024-11-16T16:15:41,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/a253513335c84b21a455bdad3ad82813 2024-11-16T16:15:41,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/3eaf9953e72243b9bea69298ef8be480, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/1ed19c903e794c8fbb95b34baa282c7b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/516fa0010177422780fe8936272cd640, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b66d53e66a0146629508ad9979903b55, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/21c9ae6f80b84985870e1c11dac574f0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f56279568c4943c395b30658bb3c0b16, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f76086613fd04414ba800c5ef85e0bd3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/543ee76847724b838c6f1d4428b45025, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c0bc2c853a644f2791a6dcb25852d2a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/2178642377f0479091eac4a2c637d12f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc] to archive 2024-11-16T16:15:41,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:15:41,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f38486fa14a04227ab0683d1757477cc 2024-11-16T16:15:41,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/92d5b0f98d3844beb4b40e05c980230e 2024-11-16T16:15:41,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/3eaf9953e72243b9bea69298ef8be480 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/3eaf9953e72243b9bea69298ef8be480 2024-11-16T16:15:41,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/7d65e2abdbe34217acc9e15cc39bb298 2024-11-16T16:15:41,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d39690898e2042f0b1119f1b746acb3b 2024-11-16T16:15:41,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/1ed19c903e794c8fbb95b34baa282c7b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/1ed19c903e794c8fbb95b34baa282c7b 2024-11-16T16:15:41,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/fbdcf6c1b5984301a3cbb4af0c1e0826 2024-11-16T16:15:41,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/26e490f9e91e46f2a7c0dafb3781692a 2024-11-16T16:15:41,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/16536a1cfca847818f8725434f485a87 2024-11-16T16:15:41,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/516fa0010177422780fe8936272cd640 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/516fa0010177422780fe8936272cd640 2024-11-16T16:15:41,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/03a924faabca4c5080530a25ced133c7 2024-11-16T16:15:41,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/05335233a5ed421a93d456bff7d81f09 2024-11-16T16:15:41,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b66d53e66a0146629508ad9979903b55 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b66d53e66a0146629508ad9979903b55 2024-11-16T16:15:41,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d960496932ba44cbb87ac2811806e454 2024-11-16T16:15:41,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/6007e88ca0b84b5ea1bff572c06d2d15 2024-11-16T16:15:41,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/68cd7070a4e845218a3a5a31b37b7f15 2024-11-16T16:15:41,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/21c9ae6f80b84985870e1c11dac574f0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/21c9ae6f80b84985870e1c11dac574f0 2024-11-16T16:15:41,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/12f47407d4e44ee9a87c86c66795996e 2024-11-16T16:15:41,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b0bee738fd32441586ec9130ac4bda5a 2024-11-16T16:15:41,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f56279568c4943c395b30658bb3c0b16 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f56279568c4943c395b30658bb3c0b16 2024-11-16T16:15:41,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/86620cd99baf484fb49a05f29512df3c 2024-11-16T16:15:41,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/664a1b2a20a644f995210cf7d71932c5 2024-11-16T16:15:41,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f76086613fd04414ba800c5ef85e0bd3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/f76086613fd04414ba800c5ef85e0bd3 2024-11-16T16:15:41,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/40d3cb907ba649ee9b7b8ebda9297fe6 2024-11-16T16:15:41,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c86561f7a0634fe290a634ae3d7ddadc 2024-11-16T16:15:41,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/543ee76847724b838c6f1d4428b45025 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/543ee76847724b838c6f1d4428b45025 2024-11-16T16:15:41,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/662ab428b4f3427bb60824dedaf6c588 2024-11-16T16:15:41,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/517c8c9e27044d3d967dbcb39d2d9510 2024-11-16T16:15:41,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c0bc2c853a644f2791a6dcb25852d2a7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c0bc2c853a644f2791a6dcb25852d2a7 2024-11-16T16:15:41,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/b83b747c331140769a31723dded4f343 2024-11-16T16:15:41,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/95a3151c906641eb82cb0fdf50fd9b3c 2024-11-16T16:15:41,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/d025f868551644fe857e7a9665a01733 2024-11-16T16:15:41,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/2178642377f0479091eac4a2c637d12f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/2178642377f0479091eac4a2c637d12f 2024-11-16T16:15:41,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/9361adc9c7dd4c4bb5e983580ca67273 2024-11-16T16:15:41,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/bcf15a221f1747009a24b4c93bdec694 2024-11-16T16:15:41,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/288b7c84c5db423fa28f06fdf5a8f578 2024-11-16T16:15:41,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/5d151fce36204f86aeca58e7526346dc 2024-11-16T16:15:41,436 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/recovered.edits/604.seqid, newMaxSeqId=604, maxSeqId=1 2024-11-16T16:15:41,436 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. 2024-11-16T16:15:41,436 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] regionserver.HRegion(1676): Region close journal for 1161dcb397ffa3f05e58f6711d26c4bb: Waiting for close lock at 1731773740068Running coprocessor pre-close hooks at 1731773740068Disabling compacts and flushes for region at 1731773740068Disabling writes for close at 1731773740068Obtaining lock to block concurrent updates at 1731773740068Preparing flush snapshotting stores in 1161dcb397ffa3f05e58f6711d26c4bb at 1731773740068Finished memstore snapshotting TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731773740069 (+1 ms)Flushing stores of TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb. at 1731773740070 (+1 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/A: creating writer at 1731773740070Flushing 1161dcb397ffa3f05e58f6711d26c4bb/A: appending metadata at 1731773740075 (+5 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/A: closing flushed file at 1731773740075Flushing 1161dcb397ffa3f05e58f6711d26c4bb/B: creating writer at 1731773740488 (+413 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/B: appending metadata at 1731773740491 (+3 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/B: closing flushed file at 1731773740491Flushing 1161dcb397ffa3f05e58f6711d26c4bb/C: creating writer at 1731773740901 (+410 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/C: appending metadata at 1731773740904 (+3 ms)Flushing 1161dcb397ffa3f05e58f6711d26c4bb/C: closing flushed file at 1731773740904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4638de85: reopening flushed file at 1731773741312 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d088a24: reopening flushed file at 1731773741316 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24894757: reopening flushed file at 1731773741320 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 1161dcb397ffa3f05e58f6711d26c4bb in 1257ms, sequenceid=601, compaction requested=false at 1731773741325 (+5 ms)Writing region close event to WAL at 1731773741433 (+108 ms)Running coprocessor post-close hooks at 1731773741436 (+3 ms)Closed at 1731773741436 2024-11-16T16:15:41,438 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=91}] handler.UnassignRegionHandler(157): Closed 1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:41,438 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=1161dcb397ffa3f05e58f6711d26c4bb, regionState=CLOSED 2024-11-16T16:15:41,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:41,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-16T16:15:41,442 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; CloseRegionProcedure 1161dcb397ffa3f05e58f6711d26c4bb, server=303fe01965e8,37355,1731773652566 in 1.5270 sec 2024-11-16T16:15:41,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-16T16:15:41,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1161dcb397ffa3f05e58f6711d26c4bb, UNASSIGN in 1.5330 sec 2024-11-16T16:15:41,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-16T16:15:41,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5380 sec 2024-11-16T16:15:41,447 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773741447"}]},"ts":"1731773741447"} 2024-11-16T16:15:41,448 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:15:41,448 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:15:41,449 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5510 sec 2024-11-16T16:15:42,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-16T16:15:42,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:42,046 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:15:42,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,048 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=92, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T16:15:42,049 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=92, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,051 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:42,054 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/recovered.edits] 2024-11-16T16:15:42,058 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/48a831504559491ba0355e73c2a68d7c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/48a831504559491ba0355e73c2a68d7c 2024-11-16T16:15:42,059 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b43177be9414793bbdf1f12cfd4a5bd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/A/8b43177be9414793bbdf1f12cfd4a5bd 2024-11-16T16:15:42,062 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6a3c7824634c4ba48e5d326afdc0d354 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/6a3c7824634c4ba48e5d326afdc0d354 2024-11-16T16:15:42,063 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/74bfbb327ed34dc88fb39090f962b9e5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/B/74bfbb327ed34dc88fb39090f962b9e5 2024-11-16T16:15:42,066 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/31b1b56fc32d414a80bea03a679e3e34 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/31b1b56fc32d414a80bea03a679e3e34 2024-11-16T16:15:42,067 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c223f8a596b1434896278537e91bda73 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/C/c223f8a596b1434896278537e91bda73 2024-11-16T16:15:42,071 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/recovered.edits/604.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb/recovered.edits/604.seqid 2024-11-16T16:15:42,072 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/1161dcb397ffa3f05e58f6711d26c4bb 2024-11-16T16:15:42,072 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:15:42,075 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=92, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,080 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:15:42,083 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:15:42,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=92, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,084 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:15:42,085 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773742084"}]},"ts":"9223372036854775807"} 2024-11-16T16:15:42,087 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:15:42,087 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1161dcb397ffa3f05e58f6711d26c4bb, NAME => 'TestAcidGuarantees,,1731773713077.1161dcb397ffa3f05e58f6711d26c4bb.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:15:42,087 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:15:42,087 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773742087"}]},"ts":"9223372036854775807"} 2024-11-16T16:15:42,088 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:15:42,089 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=92, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-16T16:15:42,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:15:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-16T16:15:42,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:15:42,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:42,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 250), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=506 (was 401) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4430 (was 3876) - AvailableMemoryMB LEAK? - 2024-11-16T16:15:42,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=506, ProcessCount=11, AvailableMemoryMB=4430 2024-11-16T16:15:42,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:15:42,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:15:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:42,175 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:15:42,175 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:42,175 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 93 2024-11-16T16:15:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:42,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:15:42,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742195_1371 (size=1061) 2024-11-16T16:15:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:42,585 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:15:42,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742196_1372 (size=53) 2024-11-16T16:15:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 951a043b6294ef712b77fdf5f51695c3, disabling compactions & flushes 2024-11-16T16:15:42,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. after waiting 0 ms 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:42,995 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:42,995 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 951a043b6294ef712b77fdf5f51695c3: Waiting for close lock at 1731773742995Disabling compacts and flushes for region at 1731773742995Disabling writes for close at 1731773742995Writing region close event to WAL at 1731773742995Closed at 1731773742995 2024-11-16T16:15:42,996 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:15:42,996 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773742996"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773742996"}]},"ts":"1731773742996"} 2024-11-16T16:15:42,998 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:15:42,999 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:15:42,999 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773742999"}]},"ts":"1731773742999"} 2024-11-16T16:15:43,000 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:15:43,001 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, ASSIGN}] 2024-11-16T16:15:43,002 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, ASSIGN 2024-11-16T16:15:43,003 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:15:43,154 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:43,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, ASSIGN because future has completed 2024-11-16T16:15:43,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:43,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:43,355 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:43,355 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7752): Opening region: {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:15:43,356 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,356 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:15:43,356 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7794): checking encryption for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,356 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(7797): checking classloading for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,359 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,360 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:43,360 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName A 2024-11-16T16:15:43,360 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:43,361 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:43,361 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,362 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:43,362 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName B 2024-11-16T16:15:43,362 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:43,362 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:43,363 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,363 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:43,364 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName C 2024-11-16T16:15:43,364 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:43,364 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:43,364 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:43,364 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1038): replaying wal for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,365 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,366 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1048): stopping wal replay for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,366 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1060): Cleaning up temporary data for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,367 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:15:43,368 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1093): writing seq id for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,369 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:15:43,370 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1114): Opened 951a043b6294ef712b77fdf5f51695c3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72672115, jitterRate=0.08289889991283417}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:15:43,370 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:43,370 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegion(1006): Region open journal for 951a043b6294ef712b77fdf5f51695c3: Running coprocessor pre-open hook at 1731773743357Writing region info on filesystem at 1731773743357Initializing all the Stores at 1731773743358 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773743358Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773743359 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773743359Cleaning up temporary data from old regions at 1731773743366 (+7 ms)Running coprocessor post-open hooks at 1731773743370 (+4 ms)Region opened successfully at 1731773743370 2024-11-16T16:15:43,371 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., pid=95, masterSystemTime=1731773743312 2024-11-16T16:15:43,373 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:43,373 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=95}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:43,374 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:43,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:43,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-16T16:15:43,377 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 in 217 msec 2024-11-16T16:15:43,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-16T16:15:43,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, ASSIGN in 376 msec 2024-11-16T16:15:43,380 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:15:43,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773743380"}]},"ts":"1731773743380"} 2024-11-16T16:15:43,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:15:43,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=93, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:15:43,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2080 sec 2024-11-16T16:15:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-16T16:15:44,325 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:44,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:44,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:44,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:44,330 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428464f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:44,333 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c975a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:44,334 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:44,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:44,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:15:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:15:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4b011027 2024-11-16T16:15:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:15:44,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:15:44,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:15:44,341 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:15:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=96, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T16:15:44,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742197_1373 (size=1097) 2024-11-16T16:15:44,754 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T16:15:44,754 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T16:15:44,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:15:44,763 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE}] 2024-11-16T16:15:44,764 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE 2024-11-16T16:15:44,765 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:44,766 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE because future has completed 2024-11-16T16:15:44,766 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:15:44,766 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:44,918 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(122): Close 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:44,919 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:15:44,919 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1722): Closing 951a043b6294ef712b77fdf5f51695c3, disabling compactions & flushes 2024-11-16T16:15:44,919 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:44,919 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:44,919 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. after waiting 0 ms 2024-11-16T16:15:44,919 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:44,922 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T16:15:44,923 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:44,923 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegion(1676): Region close journal for 951a043b6294ef712b77fdf5f51695c3: Waiting for close lock at 1731773744919Running coprocessor pre-close hooks at 1731773744919Disabling compacts and flushes for region at 1731773744919Disabling writes for close at 1731773744919Writing region close event to WAL at 1731773744919Running coprocessor post-close hooks at 1731773744923 (+4 ms)Closed at 1731773744923 2024-11-16T16:15:44,923 WARN [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] regionserver.HRegionServer(3299): Not adding moved region record: 951a043b6294ef712b77fdf5f51695c3 to self. 2024-11-16T16:15:44,925 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=99}] handler.UnassignRegionHandler(157): Closed 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:44,925 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=CLOSED 2024-11-16T16:15:44,927 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=99, ppid=98, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:44,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=99, resume processing ppid=98 2024-11-16T16:15:44,929 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, ppid=98, state=SUCCESS, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 in 161 msec 2024-11-16T16:15:44,930 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE; state=CLOSED, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=true 2024-11-16T16:15:45,080 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,082 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=97, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE because future has completed 2024-11-16T16:15:45,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:15:45,236 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,236 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7752): Opening region: {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:15:45,237 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,237 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:15:45,237 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7794): checking encryption for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,237 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(7797): checking classloading for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,238 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,239 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:45,239 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName A 2024-11-16T16:15:45,241 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:45,241 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:45,242 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,242 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:45,243 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName B 2024-11-16T16:15:45,243 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:45,243 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:45,243 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,244 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:15:45,244 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 951a043b6294ef712b77fdf5f51695c3 columnFamilyName C 2024-11-16T16:15:45,244 DEBUG [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:45,245 INFO [StoreOpener-951a043b6294ef712b77fdf5f51695c3-1 {}] regionserver.HStore(327): Store=951a043b6294ef712b77fdf5f51695c3/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:15:45,245 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,245 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1038): replaying wal for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,246 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,247 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,248 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1048): stopping wal replay for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,248 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1060): Cleaning up temporary data for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,249 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:15:45,250 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1093): writing seq id for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,251 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1114): Opened 951a043b6294ef712b77fdf5f51695c3; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72634319, jitterRate=0.0823356956243515}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:15:45,252 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,252 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegion(1006): Region open journal for 951a043b6294ef712b77fdf5f51695c3: Running coprocessor pre-open hook at 1731773745237Writing region info on filesystem at 1731773745237Initializing all the Stores at 1731773745238 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773745238Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773745238Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773745238Cleaning up temporary data from old regions at 1731773745248 (+10 ms)Running coprocessor post-open hooks at 1731773745252 (+4 ms)Region opened successfully at 1731773745252 2024-11-16T16:15:45,253 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., pid=100, masterSystemTime=1731773745234 2024-11-16T16:15:45,256 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,256 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=100}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=98 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=OPEN, openSeqNum=5, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,258 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=100, ppid=98, state=RUNNABLE, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:15:45,261 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=98 2024-11-16T16:15:45,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=98, state=SUCCESS, hasLock=false; OpenRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 in 177 msec 2024-11-16T16:15:45,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-16T16:15:45,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, REOPEN/MOVE in 499 msec 2024-11-16T16:15:45,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-16T16:15:45,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 506 msec 2024-11-16T16:15:45,272 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 928 msec 2024-11-16T16:15:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=96 2024-11-16T16:15:45,273 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:45,274 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,274 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,274 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,275 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55514fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,276 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,277 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aac664c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,279 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cddd29f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,280 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,280 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d8d033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,282 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,282 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,283 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc56b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,285 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c10a322, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,286 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,286 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,287 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eabad45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,289 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ea1a5a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,292 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74dd7215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,293 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,294 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71f45b5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,296 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e13e1c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,299 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@254f2008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,302 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,303 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108659b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,304 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2160f1e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,306 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c9ff6b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75fc4fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,308 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,308 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28f47b49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,309 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d4bf113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:15:45,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:15:45,311 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:15:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:15:45,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b8217f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:15:45,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:15:45,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@321afdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:15:45,314 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [Thread-1684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [Thread-1699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [Thread-1693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [Thread-1697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [Thread-1688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,314 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:45,315 DEBUG [Thread-1695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,315 DEBUG [Thread-1682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,316 DEBUG [Thread-1686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:15:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,316 DEBUG [Thread-1695 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,316 DEBUG [Thread-1695 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-16T16:15:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:15:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:15:45,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:45,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:45,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:15:45,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,326 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:15:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:45,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:45,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5a25931bd5e4452b7ced5fa3917ee32_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773745327/Put/seqid=0 2024-11-16T16:15:45,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742198_1374 (size=19474) 2024-11-16T16:15:45,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773805363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773805364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773745354, max timestamp=1731773745374Num uniques 859; 2024-11-16T16:15:45,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:45,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773745354, max timestamp=1731773745374Num uniques 859; 2024-11-16T16:15:45,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:45,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773745354, max timestamp=1731773745374Num uniques 859; 2024-11-16T16:15:45,378 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:15:45,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773805411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773805417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:45,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773805418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773805484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773805485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,493 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773805525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773805525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773805534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:45,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773805695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773805695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773805734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773805734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773805745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:45,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:45,769 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:45,772 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e5a25931bd5e4452b7ced5fa3917ee32_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5a25931bd5e4452b7ced5fa3917ee32_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:45,772 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4a1ccc01d2234a31963c4867027798a9, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:45,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4a1ccc01d2234a31963c4867027798a9 is 175, key is test_row_0/A:col10/1731773745327/Put/seqid=0 2024-11-16T16:15:45,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742199_1375 (size=56733) 2024-11-16T16:15:45,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:45,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:46,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773806005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773806005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773806054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773806054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773806065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:46,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4a1ccc01d2234a31963c4867027798a9 2024-11-16T16:15:46,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/98bb4f4877d047c6876cba5141e2319b is 50, key is test_row_0/B:col10/1731773745327/Put/seqid=0 2024-11-16T16:15:46,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742200_1376 (size=12001) 2024-11-16T16:15:46,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/98bb4f4877d047c6876cba5141e2319b 2024-11-16T16:15:46,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:46,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/21b42c4bb3dd4f80b85a53f3b13b4e39 is 50, key is test_row_0/C:col10/1731773745327/Put/seqid=0 2024-11-16T16:15:46,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742201_1377 (size=12001) 2024-11-16T16:15:46,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:46,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773806515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773806515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773806575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773806575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773806576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:46,654 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/21b42c4bb3dd4f80b85a53f3b13b4e39 2024-11-16T16:15:46,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4a1ccc01d2234a31963c4867027798a9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9 2024-11-16T16:15:46,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9, entries=300, sequenceid=16, filesize=55.4 K 2024-11-16T16:15:46,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/98bb4f4877d047c6876cba5141e2319b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b 2024-11-16T16:15:46,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T16:15:46,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/21b42c4bb3dd4f80b85a53f3b13b4e39 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39 2024-11-16T16:15:46,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39, entries=150, sequenceid=16, filesize=11.7 K 2024-11-16T16:15:46,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 951a043b6294ef712b77fdf5f51695c3 in 1338ms, sequenceid=16, compaction requested=false 2024-11-16T16:15:46,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:46,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-16T16:15:46,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:46,700 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:46,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:46,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e4da6854114144718a65070a391d83aa_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773745387/Put/seqid=0 2024-11-16T16:15:46,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742202_1378 (size=12154) 2024-11-16T16:15:46,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:46,718 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e4da6854114144718a65070a391d83aa_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4da6854114144718a65070a391d83aa_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:46,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c76890d8f4fa4e899b608a3f65f37b06, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:46,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c76890d8f4fa4e899b608a3f65f37b06 is 175, key is test_row_0/A:col10/1731773745387/Put/seqid=0 2024-11-16T16:15:46,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742203_1379 (size=30955) 2024-11-16T16:15:46,724 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c76890d8f4fa4e899b608a3f65f37b06 2024-11-16T16:15:46,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1fbafdba23154528bd02648bcd812b02 is 50, key is test_row_0/B:col10/1731773745387/Put/seqid=0 2024-11-16T16:15:46,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742204_1380 (size=12001) 2024-11-16T16:15:46,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:15:47,142 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1fbafdba23154528bd02648bcd812b02 2024-11-16T16:15:47,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/3d6d9c937ea64b1692d27dbf3947c64d is 50, key is test_row_0/C:col10/1731773745387/Put/seqid=0 2024-11-16T16:15:47,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742205_1381 (size=12001) 2024-11-16T16:15:47,177 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/3d6d9c937ea64b1692d27dbf3947c64d 2024-11-16T16:15:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c76890d8f4fa4e899b608a3f65f37b06 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06 2024-11-16T16:15:47,193 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06, entries=150, sequenceid=49, filesize=30.2 K 2024-11-16T16:15:47,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1fbafdba23154528bd02648bcd812b02 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02 2024-11-16T16:15:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T16:15:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/3d6d9c937ea64b1692d27dbf3947c64d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d 2024-11-16T16:15:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,210 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-16T16:15:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,212 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=0 B/0 for 951a043b6294ef712b77fdf5f51695c3 in 511ms, sequenceid=49, compaction requested=false 2024-11-16T16:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:47,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:47,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-16T16:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-16T16:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-16T16:15:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8940 sec 2024-11-16T16:15:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.9010 sec 2024-11-16T16:15:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-16T16:15:47,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,466 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:47,466 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-16T16:15:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:47,468 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:47,469 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:47,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-16T16:15:47,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:47,623 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-16T16:15:47,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:47,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169757046992ab4485a27aaaea686f3d14_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_2/A:col10/1731773747565/Put/seqid=0 2024-11-16T16:15:47,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742206_1382 (size=7274) 2024-11-16T16:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,706 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411169757046992ab4485a27aaaea686f3d14_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169757046992ab4485a27aaaea686f3d14_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:47,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/719ce248f79e4c22be67e06fd82b9c8f, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/719ce248f79e4c22be67e06fd82b9c8f is 175, key is test_row_2/A:col10/1731773747565/Put/seqid=0 2024-11-16T16:15:47,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742207_1383 (size=13765) 2024-11-16T16:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,733 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/719ce248f79e4c22be67e06fd82b9c8f 2024-11-16T16:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:47,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/b4af1f6e583b42f68375cf95b76d0edf is 50, key is test_row_2/B:col10/1731773747565/Put/seqid=0 2024-11-16T16:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742208_1384 (size=7315) 2024-11-16T16:15:47,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:47,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:47,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773747765, max timestamp=1731773747855Num uniques 859; 2024-11-16T16:15:47,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773747765, max timestamp=1731773747855Num uniques 859; 2024-11-16T16:15:47,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:47,859 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773747765, max timestamp=1731773747855Num uniques 859; 2024-11-16T16:15:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:15:47,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731773747904, max timestamp=1731773747923Num uniques 1018; 2024-11-16T16:15:47,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731773747904, max timestamp=1731773747923Num uniques 1018; 2024-11-16T16:15:47,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T16:15:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731773747904, max timestamp=1731773747923Num uniques 1018; 2024-11-16T16:15:47,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T16:15:47,927 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-16T16:15:47,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731773747948, max timestamp=1731773747979Num uniques 1077; 2024-11-16T16:15:47,985 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-16T16:15:47,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731773747948, max timestamp=1731773747979Num uniques 1077; 2024-11-16T16:15:47,989 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=6870, suffix heap size=139976,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=150 2024-11-16T16:15:47,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731773747948, max timestamp=1731773747979Num uniques 1077; 2024-11-16T16:15:47,990 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731773748033, max timestamp=1731773748056Num uniques 1186; 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731773748033, max timestamp=1731773748056Num uniques 1009; 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731773748033, max timestamp=1731773748056Num uniques 1186; 2024-11-16T16:15:48,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-16T16:15:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:48,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731773748079, max timestamp=1731773748130Num uniques 1095; 2024-11-16T16:15:48,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731773748079, max timestamp=1731773748130Num uniques 1095; 2024-11-16T16:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-16T16:15:48,135 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-16T16:15:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1731773748079, max timestamp=1731773748130Num uniques 1095; 2024-11-16T16:15:48,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-16T16:15:48,180 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/b4af1f6e583b42f68375cf95b76d0edf 2024-11-16T16:15:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/df3ce15e31ae4f01addaf8ce5bd2c21a is 50, key is test_row_2/C:col10/1731773747565/Put/seqid=0 2024-11-16T16:15:48,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742209_1385 (size=7315) 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731773748186, max timestamp=1731773748232Num uniques 1204; 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731773748186, max timestamp=1731773748232Num uniques 1204; 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731773748186, max timestamp=1731773748232Num uniques 1204; 2024-11-16T16:15:48,239 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=55140, new segment data size=9340, suffix heap size=153184,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1204, new segment cells count=204 2024-11-16T16:15:48,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9738, totalHeapSize=22392, min timestamp=1731773748287, max timestamp=1731773748328Num uniques 1063; 2024-11-16T16:15:48,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48668, new segment data size=9738, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-16T16:15:48,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9738, totalHeapSize=22392, min timestamp=1731773748287, max timestamp=1731773748328Num uniques 1063; 2024-11-16T16:15:48,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48668, new segment data size=9738, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-16T16:15:48,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=171, cellSize=7826, totalHeapSize=18024, min timestamp=1731773748287, max timestamp=1731773748328Num uniques 1063; 2024-11-16T16:15:48,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48668, new segment data size=7826, suffix heap size=138520,new segment heap size=18024 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=171 2024-11-16T16:15:48,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731773748390, max timestamp=1731773748410Num uniques 1072; 2024-11-16T16:15:48,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T16:15:48,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731773748390, max timestamp=1731773748410Num uniques 1072; 2024-11-16T16:15:48,420 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-16T16:15:48,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731773748390, max timestamp=1731773748410Num uniques 1030; 2024-11-16T16:15:48,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47180, new segment data size=7872, suffix heap size=135088,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1030, new segment cells count=172 2024-11-16T16:15:48,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731773748454, max timestamp=1731773748468Num uniques 1031; 2024-11-16T16:15:48,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T16:15:48,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=331, cellSize=15156, totalHeapSize=34664, min timestamp=1731773748442, max timestamp=1731773748468Num uniques 1031; 2024-11-16T16:15:48,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=15156, suffix heap size=135192,new segment heap size=34664 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=331 2024-11-16T16:15:48,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1731773748454, max timestamp=1731773748468Num uniques 1031; 2024-11-16T16:15:48,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-11-16T16:15:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731773748468, max timestamp=1731773748524Num uniques 1190; 2024-11-16T16:15:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54500, new segment data size=13280, suffix heap size=151728,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1190, new segment cells count=290 2024-11-16T16:15:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731773748468, max timestamp=1731773748524Num uniques 1090; 2024-11-16T16:15:48,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-16T16:15:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=290, cellSize=13280, totalHeapSize=30400, min timestamp=1731773748468, max timestamp=1731773748524Num uniques 1090; 2024-11-16T16:15:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=13280, suffix heap size=141328,new segment heap size=30400 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=290 2024-11-16T16:15:48,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9105, totalHeapSize=20936, min timestamp=1731773748563, max timestamp=1731773748580Num uniques 1149; 2024-11-16T16:15:48,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1731773748563, max timestamp=1731773748580Num uniques 1149; 2024-11-16T16:15:48,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52615, new segment data size=9105, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T16:15:48,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52614, new segment data size=9104, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T16:15:48,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9113, totalHeapSize=20936, min timestamp=1731773748563, max timestamp=1731773748580Num uniques 1149; 2024-11-16T16:15:48,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52623, new segment data size=9113, suffix heap size=147464,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1149, new segment cells count=199 2024-11-16T16:15:48,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:48,635 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/df3ce15e31ae4f01addaf8ce5bd2c21a 2024-11-16T16:15:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/719ce248f79e4c22be67e06fd82b9c8f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f 2024-11-16T16:15:48,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731773748621, max timestamp=1731773748647Num uniques 1058; 2024-11-16T16:15:48,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731773748621, max timestamp=1731773748647Num uniques 1058; 2024-11-16T16:15:48,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-11-16T16:15:48,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-11-16T16:15:48,652 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f, entries=50, sequenceid=53, filesize=13.4 K 2024-11-16T16:15:48,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/b4af1f6e583b42f68375cf95b76d0edf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf 2024-11-16T16:15:48,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=308, cellSize=14100, totalHeapSize=32272, min timestamp=1731773748621, max timestamp=1731773748647Num uniques 1058; 2024-11-16T16:15:48,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=14100, suffix heap size=138000,new segment heap size=32272 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=308 2024-11-16T16:15:48,658 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf, entries=50, sequenceid=53, filesize=7.1 K 2024-11-16T16:15:48,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/df3ce15e31ae4f01addaf8ce5bd2c21a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a 2024-11-16T16:15:48,663 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a, entries=50, sequenceid=53, filesize=7.1 K 2024-11-16T16:15:48,664 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=53.67 KB/54960 for 951a043b6294ef712b77fdf5f51695c3 in 1041ms, sequenceid=53, compaction requested=true 2024-11-16T16:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:48,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-16T16:15:48,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-16T16:15:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:48,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=73.80 KB heapSize=191.72 KB 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:48,670 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:48,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-16T16:15:48,672 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1980 sec 2024-11-16T16:15:48,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2070 sec 2024-11-16T16:15:48,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111639d017a8dd31432aab43028386a0f447_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:48,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742210_1386 (size=12454) 2024-11-16T16:15:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773808710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773808711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773808725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773808726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773808727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773808834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773808834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773808835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773808845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:48,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773808845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:48,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773809055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773809055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773809055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773809065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773809065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,107 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:49,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111639d017a8dd31432aab43028386a0f447_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111639d017a8dd31432aab43028386a0f447_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:49,111 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/37de7596c5044ddea2bc79154492e2e6, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:49,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/37de7596c5044ddea2bc79154492e2e6 is 175, key is test_row_0/A:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:49,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742211_1387 (size=31255) 2024-11-16T16:15:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773809375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773809375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773809375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773809385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773809385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/37de7596c5044ddea2bc79154492e2e6 2024-11-16T16:15:49,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/0d336eea7f9b4eb6b946d33f63c4d5ab is 50, key is test_row_0/B:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:49,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742212_1388 (size=12301) 2024-11-16T16:15:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-16T16:15:49,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:49,626 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-16T16:15:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:49,627 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:49,628 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:49,628 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:49,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T16:15:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:49,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773809885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773809885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773809886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773809895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773809905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:49,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T16:15:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:49,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:49,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/0d336eea7f9b4eb6b946d33f63c4d5ab 2024-11-16T16:15:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:49,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f22901fdd46047cb9f256eee5edccaa8 is 50, key is test_row_0/C:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:49,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742213_1389 (size=12301) 2024-11-16T16:15:50,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T16:15:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T16:15:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:50,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f22901fdd46047cb9f256eee5edccaa8 2024-11-16T16:15:50,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/37de7596c5044ddea2bc79154492e2e6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6 2024-11-16T16:15:50,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6, entries=150, sequenceid=267, filesize=30.5 K 2024-11-16T16:15:50,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/0d336eea7f9b4eb6b946d33f63c4d5ab as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab 2024-11-16T16:15:50,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T16:15:50,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f22901fdd46047cb9f256eee5edccaa8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8 2024-11-16T16:15:50,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8, entries=150, sequenceid=267, filesize=12.0 K 2024-11-16T16:15:50,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~191.67 KB/196272, currentSize=114.05 KB/116790 for 951a043b6294ef712b77fdf5f51695c3 in 1696ms, sequenceid=267, compaction requested=true 2024-11-16T16:15:50,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:50,366 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:50,366 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:50,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:50,366 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132708 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:50,366 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:15:50,367 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,367 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,367 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=129.6 K 2024-11-16T16:15:50,367 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=42.6 K 2024-11-16T16:15:50,367 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6] 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1ccc01d2234a31963c4867027798a9, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731773745325 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 98bb4f4877d047c6876cba5141e2319b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731773745327 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c76890d8f4fa4e899b608a3f65f37b06, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773745387 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fbafdba23154528bd02648bcd812b02, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773745387 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 719ce248f79e4c22be67e06fd82b9c8f, keycount=50, bloomtype=ROW, size=13.4 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731773747565 2024-11-16T16:15:50,367 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b4af1f6e583b42f68375cf95b76d0edf, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731773747565 2024-11-16T16:15:50,368 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d336eea7f9b4eb6b946d33f63c4d5ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:50,368 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37de7596c5044ddea2bc79154492e2e6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:50,373 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:50,375 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411162540a6e8ec7f415596c15e5807c6a802_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:50,377 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#338 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:50,377 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/e04a8a6328e647b09ec08b5d85baa2c7 is 50, key is test_row_0/B:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:50,377 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411162540a6e8ec7f415596c15e5807c6a802_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:50,377 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411162540a6e8ec7f415596c15e5807c6a802_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:50,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742214_1390 (size=12439) 2024-11-16T16:15:50,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742215_1391 (size=4469) 2024-11-16T16:15:50,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-16T16:15:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,392 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:50,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d1947738dc9b446cbe68c66809674d80_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773748705/Put/seqid=0 2024-11-16T16:15:50,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742216_1392 (size=12454) 2024-11-16T16:15:50,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:50,786 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#339 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:50,786 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/d5b1b55f9a1646a3ab30f1c432727906 is 175, key is test_row_0/A:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:50,788 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/e04a8a6328e647b09ec08b5d85baa2c7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/e04a8a6328e647b09ec08b5d85baa2c7 2024-11-16T16:15:50,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742217_1393 (size=31393) 2024-11-16T16:15:50,792 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into e04a8a6328e647b09ec08b5d85baa2c7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:50,792 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:50,792 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=12, startTime=1731773750366; duration=0sec 2024-11-16T16:15:50,792 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:50,792 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:15:50,792 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:50,793 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 43618 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:50,793 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:15:50,793 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:50,793 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=42.6 K 2024-11-16T16:15:50,794 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 21b42c4bb3dd4f80b85a53f3b13b4e39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731773745327 2024-11-16T16:15:50,794 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d6d9c937ea64b1692d27dbf3947c64d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731773745387 2024-11-16T16:15:50,794 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting df3ce15e31ae4f01addaf8ce5bd2c21a, keycount=50, bloomtype=ROW, size=7.1 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731773747565 2024-11-16T16:15:50,794 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f22901fdd46047cb9f256eee5edccaa8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:50,801 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:50,801 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/ecdfc727d828428f98c009e5a4c6cfc7 is 50, key is test_row_0/C:col10/1731773748649/Put/seqid=0 2024-11-16T16:15:50,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:50,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742218_1394 (size=12439) 2024-11-16T16:15:50,806 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d1947738dc9b446cbe68c66809674d80_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d1947738dc9b446cbe68c66809674d80_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:50,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/417e918f76ae4a32aa0ead1977130724, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:50,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/417e918f76ae4a32aa0ead1977130724 is 175, key is test_row_0/A:col10/1731773748705/Put/seqid=0 2024-11-16T16:15:50,810 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/ecdfc727d828428f98c009e5a4c6cfc7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/ecdfc727d828428f98c009e5a4c6cfc7 2024-11-16T16:15:50,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742219_1395 (size=31255) 2024-11-16T16:15:50,815 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into ecdfc727d828428f98c009e5a4c6cfc7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:50,815 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:50,815 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=12, startTime=1731773750366; duration=0sec 2024-11-16T16:15:50,815 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:50,815 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:15:50,816 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/417e918f76ae4a32aa0ead1977130724 2024-11-16T16:15:50,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/dfc4b1ec569a43e98ae57159e8903307 is 50, key is test_row_0/B:col10/1731773748705/Put/seqid=0 2024-11-16T16:15:50,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742220_1396 (size=12301) 2024-11-16T16:15:50,830 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/dfc4b1ec569a43e98ae57159e8903307 2024-11-16T16:15:50,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a3790a0860eb4e848833bc21dc079bfc is 50, key is test_row_0/C:col10/1731773748705/Put/seqid=0 2024-11-16T16:15:50,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742221_1397 (size=12301) 2024-11-16T16:15:50,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:50,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:50,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:50,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773810915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773810916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773810917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773810918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:50,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773810925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:50,932 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:50,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773811025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773811035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773811035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773811035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,194 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/d5b1b55f9a1646a3ab30f1c432727906 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906 2024-11-16T16:15:51,197 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into d5b1b55f9a1646a3ab30f1c432727906(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:51,197 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:51,197 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=12, startTime=1731773750365; duration=0sec 2024-11-16T16:15:51,197 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:51,198 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:15:51,241 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a3790a0860eb4e848833bc21dc079bfc 2024-11-16T16:15:51,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/417e918f76ae4a32aa0ead1977130724 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724 2024-11-16T16:15:51,247 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724, entries=150, sequenceid=287, filesize=30.5 K 2024-11-16T16:15:51,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/dfc4b1ec569a43e98ae57159e8903307 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307 2024-11-16T16:15:51,251 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T16:15:51,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a3790a0860eb4e848833bc21dc079bfc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc 2024-11-16T16:15:51,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773811245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,255 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc, entries=150, sequenceid=287, filesize=12.0 K 2024-11-16T16:15:51,256 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 951a043b6294ef712b77fdf5f51695c3 in 863ms, sequenceid=287, compaction requested=false 2024-11-16T16:15:51,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:51,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:51,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-16T16:15:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-16T16:15:51,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-16T16:15:51,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-16T16:15:51,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.6330 sec 2024-11-16T16:15:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:51,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:51,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:51,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:51,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161bbdb7649c1f4dadbf7fe569b5ec394c_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:51,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742222_1398 (size=14994) 2024-11-16T16:15:51,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773811376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773811495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773811565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773811565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773811565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,681 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:51,684 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161bbdb7649c1f4dadbf7fe569b5ec394c_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bbdb7649c1f4dadbf7fe569b5ec394c_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:51,684 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/3017a25fd2214d14a5b7fea25fda0e59, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:51,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/3017a25fd2214d14a5b7fea25fda0e59 is 175, key is test_row_0/A:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:51,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742223_1399 (size=39949) 2024-11-16T16:15:51,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773811715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:51,728 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-16T16:15:51,775 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:51,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-16T16:15:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:51,777 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:51,778 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:51,778 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:51,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T16:15:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:51,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:51,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773812035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T16:15:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773812075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773812075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/3017a25fd2214d14a5b7fea25fda0e59 2024-11-16T16:15:52,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773812075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:52,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/7a36246b60ca4abe87939911a97b7ab3 is 50, key is test_row_0/B:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742224_1400 (size=12301) 2024-11-16T16:15:52,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/7a36246b60ca4abe87939911a97b7ab3 2024-11-16T16:15:52,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 is 50, key is test_row_0/C:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:52,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742225_1401 (size=12301) 2024-11-16T16:15:52,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:15:52,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T16:15:52,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T16:15:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T16:15:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:52,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 2024-11-16T16:15:52,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/3017a25fd2214d14a5b7fea25fda0e59 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59 2024-11-16T16:15:52,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59, entries=200, sequenceid=303, filesize=39.0 K 2024-11-16T16:15:52,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/7a36246b60ca4abe87939911a97b7ab3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3 2024-11-16T16:15:52,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T16:15:52,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 2024-11-16T16:15:52,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5, entries=150, sequenceid=303, filesize=12.0 K 2024-11-16T16:15:52,529 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 951a043b6294ef712b77fdf5f51695c3 in 1267ms, sequenceid=303, compaction requested=true 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:52,529 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:52,529 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102597 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:15:52,530 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,530 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,530 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=100.2 K 2024-11-16T16:15:52,530 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/e04a8a6328e647b09ec08b5d85baa2c7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.2 K 2024-11-16T16:15:52,530 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59] 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e04a8a6328e647b09ec08b5d85baa2c7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:52,530 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5b1b55f9a1646a3ab30f1c432727906, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:52,531 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting dfc4b1ec569a43e98ae57159e8903307, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731773748705 2024-11-16T16:15:52,531 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417e918f76ae4a32aa0ead1977130724, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731773748705 2024-11-16T16:15:52,531 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a36246b60ca4abe87939911a97b7ab3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:52,531 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3017a25fd2214d14a5b7fea25fda0e59, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:52,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,542 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:52,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:52,547 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:52,548 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/acf784a609a044e7be7cf074d86af233 is 50, key is test_row_0/B:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:52,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:52,549 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:52,557 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111628d17d4025eb43d3b78e316fdac8b6e1_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:52,559 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111628d17d4025eb43d3b78e316fdac8b6e1_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c9bd9d41040459cbb14b525015d69c7_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773751315/Put/seqid=0 2024-11-16T16:15:52,559 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111628d17d4025eb43d3b78e316fdac8b6e1_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:52,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742226_1402 (size=12541) 2024-11-16T16:15:52,567 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/acf784a609a044e7be7cf074d86af233 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/acf784a609a044e7be7cf074d86af233 2024-11-16T16:15:52,572 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into acf784a609a044e7be7cf074d86af233(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:52,572 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:52,572 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=13, startTime=1731773752529; duration=0sec 2024-11-16T16:15:52,572 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:52,572 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:15:52,572 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:15:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742228_1404 (size=4469) 2024-11-16T16:15:52,578 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:15:52,578 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:15:52,578 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:52,578 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/ecdfc727d828428f98c009e5a4c6cfc7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.2 K 2024-11-16T16:15:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742227_1403 (size=12454) 2024-11-16T16:15:52,579 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdfc727d828428f98c009e5a4c6cfc7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731773748649 2024-11-16T16:15:52,579 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a3790a0860eb4e848833bc21dc079bfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731773748705 2024-11-16T16:15:52,579 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 01e49c7bf08f46fba2b4dbfc8b6ad8f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:52,591 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:52,592 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/8140772a5c934c59a8caa8118fdc1735 is 50, key is test_row_0/C:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742229_1405 (size=12541) 2024-11-16T16:15:52,604 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/8140772a5c934c59a8caa8118fdc1735 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8140772a5c934c59a8caa8118fdc1735 2024-11-16T16:15:52,608 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into 8140772a5c934c59a8caa8118fdc1735(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:52,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:52,608 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=13, startTime=1731773752529; duration=0sec 2024-11-16T16:15:52,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:52,608 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:15:52,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773812627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,634 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773812745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:52,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773812945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,953 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,954 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:52,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773812955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:52,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:52,979 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#348 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:52,980 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/89e2695271a84548878feeb796c2bb61 is 175, key is test_row_0/A:col10/1731773751261/Put/seqid=0 2024-11-16T16:15:52,982 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c9bd9d41040459cbb14b525015d69c7_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c9bd9d41040459cbb14b525015d69c7_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:52,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/df39f14cca2d4188968a4d635ec31aac, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:52,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/df39f14cca2d4188968a4d635ec31aac is 175, key is test_row_0/A:col10/1731773751315/Put/seqid=0 2024-11-16T16:15:52,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742230_1406 (size=31495) 2024-11-16T16:15:52,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742231_1407 (size=31255) 2024-11-16T16:15:52,993 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/df39f14cca2d4188968a4d635ec31aac 2024-11-16T16:15:52,993 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/89e2695271a84548878feeb796c2bb61 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61 2024-11-16T16:15:53,001 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into 89e2695271a84548878feeb796c2bb61(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:53,001 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:53,001 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=13, startTime=1731773752529; duration=0sec 2024-11-16T16:15:53,001 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:53,001 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:15:53,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/47c0b7a0904d43b2ab781504fba74db3 is 50, key is test_row_0/B:col10/1731773751315/Put/seqid=0 2024-11-16T16:15:53,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742232_1408 (size=12301) 2024-11-16T16:15:53,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:53,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773813096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:53,101 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773813105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:53,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773813105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:53,113 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:53,113 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:53,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773813275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:53,412 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/47c0b7a0904d43b2ab781504fba74db3 2024-11-16T16:15:53,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/fbc896a9a6bd422480189c618fc465fe is 50, key is test_row_0/C:col10/1731773751315/Put/seqid=0 2024-11-16T16:15:53,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742233_1409 (size=12301) 2024-11-16T16:15:53,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:53,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773813785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:53,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:53,825 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/fbc896a9a6bd422480189c618fc465fe 2024-11-16T16:15:53,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/df39f14cca2d4188968a4d635ec31aac as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac 2024-11-16T16:15:53,832 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac, entries=150, sequenceid=322, filesize=30.5 K 2024-11-16T16:15:53,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/47c0b7a0904d43b2ab781504fba74db3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3 2024-11-16T16:15:53,837 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3, entries=150, sequenceid=322, filesize=12.0 K 2024-11-16T16:15:53,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/fbc896a9a6bd422480189c618fc465fe as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe 2024-11-16T16:15:53,840 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe, entries=150, sequenceid=322, filesize=12.0 K 2024-11-16T16:15:53,841 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 951a043b6294ef712b77fdf5f51695c3 in 1299ms, sequenceid=322, compaction requested=false 2024-11-16T16:15:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:53,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-16T16:15:53,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-16T16:15:53,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-16T16:15:53,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0640 sec 2024-11-16T16:15:53,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.0690 sec 2024-11-16T16:15:53,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-16T16:15:53,925 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:53,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-16T16:15:53,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:53,927 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:53,928 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:53,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:54,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:54,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-16T16:15:54,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:54,080 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:54,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:54,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae2b8ec72ee8486899c1009a4d496df3_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773752609/Put/seqid=0 2024-11-16T16:15:54,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742234_1410 (size=12454) 2024-11-16T16:15:54,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:54,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:54,506 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ae2b8ec72ee8486899c1009a4d496df3_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae2b8ec72ee8486899c1009a4d496df3_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:54,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c3f0825c76c545e08af63657dab3e887, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:54,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c3f0825c76c545e08af63657dab3e887 is 175, key is test_row_0/A:col10/1731773752609/Put/seqid=0 2024-11-16T16:15:54,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742235_1411 (size=31255) 2024-11-16T16:15:54,520 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c3f0825c76c545e08af63657dab3e887 2024-11-16T16:15:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/55cdd62daf87460294477696cf7e4a98 is 50, key is test_row_0/B:col10/1731773752609/Put/seqid=0 2024-11-16T16:15:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:54,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742236_1412 (size=12301) 2024-11-16T16:15:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:54,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:54,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773814906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:54,912 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:54,957 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/55cdd62daf87460294477696cf7e4a98 2024-11-16T16:15:54,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f7d71dfe36904b359ea85c55b1b01920 is 50, key is test_row_0/C:col10/1731773752609/Put/seqid=0 2024-11-16T16:15:54,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742237_1413 (size=12301) 2024-11-16T16:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773815014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:55,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773815115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,119 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773815125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,134 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:55,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773815135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,140 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773815225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,367 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f7d71dfe36904b359ea85c55b1b01920 2024-11-16T16:15:55,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/c3f0825c76c545e08af63657dab3e887 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887 2024-11-16T16:15:55,373 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887, entries=150, sequenceid=338, filesize=30.5 K 2024-11-16T16:15:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/55cdd62daf87460294477696cf7e4a98 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98 2024-11-16T16:15:55,379 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T16:15:55,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/f7d71dfe36904b359ea85c55b1b01920 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920 2024-11-16T16:15:55,384 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920, entries=150, sequenceid=338, filesize=12.0 K 2024-11-16T16:15:55,385 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 951a043b6294ef712b77fdf5f51695c3 in 1305ms, sequenceid=338, compaction requested=true 2024-11-16T16:15:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:55,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-16T16:15:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-16T16:15:55,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-16T16:15:55,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4600 sec 2024-11-16T16:15:55,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4680 sec 2024-11-16T16:15:55,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:55,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:55,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:55,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ba31850daa72492a9004ddeebc3c0953_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:55,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742238_1414 (size=14994) 2024-11-16T16:15:55,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773815636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773815755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,965 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:55,968 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ba31850daa72492a9004ddeebc3c0953_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ba31850daa72492a9004ddeebc3c0953_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:55,969 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/99d120044e04434687ecc35d91caa05d, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:55,969 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/99d120044e04434687ecc35d91caa05d is 175, key is test_row_0/A:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:55,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:55,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773815964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:55,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:55,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742239_1415 (size=39949) 2024-11-16T16:15:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-16T16:15:56,085 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:15:56,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:15:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-16T16:15:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:56,087 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:15:56,087 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:15:56,087 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:15:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:56,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773816275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:56,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/99d120044e04434687ecc35d91caa05d 2024-11-16T16:15:56,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9f5f597f91a74394a946eb8b146ed8fe is 50, key is test_row_0/B:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:56,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742240_1416 (size=12301) 2024-11-16T16:15:56,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:56,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:56,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:56,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,545 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:56,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:56,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:56,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9f5f597f91a74394a946eb8b146ed8fe 2024-11-16T16:15:56,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/113701ba5c56400f8a555df24b80c468 is 50, key is test_row_0/C:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:56,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742241_1417 (size=12301) 2024-11-16T16:15:56,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773816795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:56,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:56,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:56,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45388 deadline: 1731773816985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:56,992 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:56,993 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:57,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:57,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:57,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:57,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:57,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:15:57,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/113701ba5c56400f8a555df24b80c468 2024-11-16T16:15:57,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/99d120044e04434687ecc35d91caa05d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d 2024-11-16T16:15:57,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d, entries=200, sequenceid=358, filesize=39.0 K 2024-11-16T16:15:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9f5f597f91a74394a946eb8b146ed8fe as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe 2024-11-16T16:15:57,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T16:15:57,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/113701ba5c56400f8a555df24b80c468 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468 2024-11-16T16:15:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468, entries=150, sequenceid=358, filesize=12.0 K 2024-11-16T16:15:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 951a043b6294ef712b77fdf5f51695c3 in 1652ms, sequenceid=358, compaction requested=true 2024-11-16T16:15:57,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:57,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:15:57,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:57,205 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:57,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:15:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:15:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:57,206 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:57,206 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:15:57,207 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,207 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,207 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/acf784a609a044e7be7cf074d86af233, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=48.3 K 2024-11-16T16:15:57,207 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=130.8 K 2024-11-16T16:15:57,207 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d] 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting acf784a609a044e7be7cf074d86af233, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e2695271a84548878feeb796c2bb61, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 47c0b7a0904d43b2ab781504fba74db3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731773751315 2024-11-16T16:15:57,207 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting df39f14cca2d4188968a4d635ec31aac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731773751315 2024-11-16T16:15:57,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 55cdd62daf87460294477696cf7e4a98, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731773752560 2024-11-16T16:15:57,208 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f0825c76c545e08af63657dab3e887, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731773752560 2024-11-16T16:15:57,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f5f597f91a74394a946eb8b146ed8fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754892 2024-11-16T16:15:57,208 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99d120044e04434687ecc35d91caa05d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754885 2024-11-16T16:15:57,213 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:57,215 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#360 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:57,215 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/93837b9623a34c3c9724f4e8ad8f6e19 is 50, key is test_row_0/B:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:57,217 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116591d7953dbc144d99ed12dfb822e6ce6_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:57,220 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116591d7953dbc144d99ed12dfb822e6ce6_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:57,220 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116591d7953dbc144d99ed12dfb822e6ce6_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:57,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742242_1418 (size=12677) 2024-11-16T16:15:57,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742243_1419 (size=4469) 2024-11-16T16:15:57,228 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#359 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:57,228 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/527435370e6a495f949fb8f77639ddc4 is 175, key is test_row_0/A:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742244_1420 (size=31631) 2024-11-16T16:15:57,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-16T16:15:57,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,307 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:15:57,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:57,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:57,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:57,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:57,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:57,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:15:57,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160bacf44e7d634ac2bb47bf8cef32281d_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773755625/Put/seqid=0 2024-11-16T16:15:57,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742245_1421 (size=12454) 2024-11-16T16:15:57,631 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/93837b9623a34c3c9724f4e8ad8f6e19 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/93837b9623a34c3c9724f4e8ad8f6e19 2024-11-16T16:15:57,639 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/527435370e6a495f949fb8f77639ddc4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4 2024-11-16T16:15:57,639 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into 93837b9623a34c3c9724f4e8ad8f6e19(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:57,639 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:57,639 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=12, startTime=1731773757205; duration=0sec 2024-11-16T16:15:57,639 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:15:57,639 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:15:57,640 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:15:57,640 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49444 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:15:57,641 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:15:57,641 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:57,641 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8140772a5c934c59a8caa8118fdc1735, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=48.3 K 2024-11-16T16:15:57,641 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8140772a5c934c59a8caa8118fdc1735, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731773750911 2024-11-16T16:15:57,642 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting fbc896a9a6bd422480189c618fc465fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731773751315 2024-11-16T16:15:57,642 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f7d71dfe36904b359ea85c55b1b01920, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731773752560 2024-11-16T16:15:57,642 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 113701ba5c56400f8a555df24b80c468, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754892 2024-11-16T16:15:57,644 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into 527435370e6a495f949fb8f77639ddc4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:57,644 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:57,644 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=12, startTime=1731773757205; duration=0sec 2024-11-16T16:15:57,644 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:57,644 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:15:57,651 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:15:57,651 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/d80ca39ccf6546a594232c81067c49f6 is 50, key is test_row_0/C:col10/1731773754892/Put/seqid=0 2024-11-16T16:15:57,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742246_1422 (size=12677) 2024-11-16T16:15:57,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:57,719 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160bacf44e7d634ac2bb47bf8cef32281d_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160bacf44e7d634ac2bb47bf8cef32281d_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:57,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4530751bbc51490da4a89f238e18a53e, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:57,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4530751bbc51490da4a89f238e18a53e is 175, key is test_row_0/A:col10/1731773755625/Put/seqid=0 2024-11-16T16:15:57,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742247_1423 (size=31255) 2024-11-16T16:15:57,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:57,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:15:58,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773817998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:58,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/d80ca39ccf6546a594232c81067c49f6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d80ca39ccf6546a594232c81067c49f6 2024-11-16T16:15:58,081 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into d80ca39ccf6546a594232c81067c49f6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:15:58,081 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:58,081 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=12, startTime=1731773757206; duration=0sec 2024-11-16T16:15:58,081 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:15:58,081 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:15:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773818105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:58,125 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4530751bbc51490da4a89f238e18a53e 2024-11-16T16:15:58,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/57d84e6c5f634bb8a8da83842576e164 is 50, key is test_row_0/B:col10/1731773755625/Put/seqid=0 2024-11-16T16:15:58,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742248_1424 (size=12301) 2024-11-16T16:15:58,148 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/57d84e6c5f634bb8a8da83842576e164 2024-11-16T16:15:58,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/77eeb970431e4961b65bd96fcb42a9e3 is 50, key is test_row_0/C:col10/1731773755625/Put/seqid=0 2024-11-16T16:15:58,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742249_1425 (size=12301) 2024-11-16T16:15:58,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:15:58,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773818315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:58,571 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/77eeb970431e4961b65bd96fcb42a9e3 2024-11-16T16:15:58,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/4530751bbc51490da4a89f238e18a53e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e 2024-11-16T16:15:58,580 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e, entries=150, sequenceid=370, filesize=30.5 K 2024-11-16T16:15:58,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/57d84e6c5f634bb8a8da83842576e164 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164 2024-11-16T16:15:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T16:15:58,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/77eeb970431e4961b65bd96fcb42a9e3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3 2024-11-16T16:15:58,597 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T16:15:58,598 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 951a043b6294ef712b77fdf5f51695c3 in 1291ms, sequenceid=370, compaction requested=false 2024-11-16T16:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-16T16:15:58,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-16T16:15:58,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-16T16:15:58,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-11-16T16:15:58,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.5180 sec 2024-11-16T16:15:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:58,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:15:58,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:15:58,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165fd6d6b7e87c468a92e1c867a01fe660_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773758627/Put/seqid=0 2024-11-16T16:15:58,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742250_1426 (size=14994) 2024-11-16T16:15:58,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:58,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773818702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:58,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773818815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773819025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:15:59,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165fd6d6b7e87c468a92e1c867a01fe660_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fd6d6b7e87c468a92e1c867a01fe660_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:15:59,050 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/45dc5796b66a4af7b97ad630259efe9c, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:15:59,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/45dc5796b66a4af7b97ad630259efe9c is 175, key is test_row_0/A:col10/1731773758627/Put/seqid=0 2024-11-16T16:15:59,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742251_1427 (size=39949) 2024-11-16T16:15:59,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45340 deadline: 1731773819134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,141 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45370 deadline: 1731773819145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,154 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45400 deadline: 1731773819185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,193 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 951a043b6294ef712b77fdf5f51695c3 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:15:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773819344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,352 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,454 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=394, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/45dc5796b66a4af7b97ad630259efe9c 2024-11-16T16:15:59,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/047c910dfa194416a3d442ccacd16cab is 50, key is test_row_0/B:col10/1731773758627/Put/seqid=0 2024-11-16T16:15:59,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742252_1428 (size=12301) 2024-11-16T16:15:59,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/047c910dfa194416a3d442ccacd16cab 2024-11-16T16:15:59,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/8e4eb618dd784c64ba03f4ba9e3f16a3 is 50, key is test_row_0/C:col10/1731773758627/Put/seqid=0 2024-11-16T16:15:59,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:15:59,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773819865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:15:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:15:59,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:15:59,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742253_1429 (size=12301) 2024-11-16T16:16:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-16T16:16:00,255 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:00,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-16T16:16:00,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:00,257 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:00,257 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:00,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:00,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=394 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/8e4eb618dd784c64ba03f4ba9e3f16a3 2024-11-16T16:16:00,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/45dc5796b66a4af7b97ad630259efe9c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c 2024-11-16T16:16:00,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c, entries=200, sequenceid=394, filesize=39.0 K 2024-11-16T16:16:00,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/047c910dfa194416a3d442ccacd16cab as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab 2024-11-16T16:16:00,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab, entries=150, sequenceid=394, filesize=12.0 K 2024-11-16T16:16:00,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/8e4eb618dd784c64ba03f4ba9e3f16a3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3 2024-11-16T16:16:00,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3, entries=150, sequenceid=394, filesize=12.0 K 2024-11-16T16:16:00,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 951a043b6294ef712b77fdf5f51695c3 in 1672ms, sequenceid=394, compaction requested=true 2024-11-16T16:16:00,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:00,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:00,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:00,302 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:00,302 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:00,303 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:00,303 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:00,303 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:00,303 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:16:00,303 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:16:00,303 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:00,303 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:00,303 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/93837b9623a34c3c9724f4e8ad8f6e19, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.4 K 2024-11-16T16:16:00,303 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=100.4 K 2024-11-16T16:16:00,303 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:00,303 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c] 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 93837b9623a34c3c9724f4e8ad8f6e19, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754892 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 527435370e6a495f949fb8f77639ddc4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754892 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d84e6c5f634bb8a8da83842576e164, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773755608 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4530751bbc51490da4a89f238e18a53e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773755608 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 047c910dfa194416a3d442ccacd16cab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:00,304 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45dc5796b66a4af7b97ad630259efe9c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:00,308 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:00,309 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#368 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:00,310 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/0defc587b0c44c37baa8c05bfaa7cfea is 50, key is test_row_0/B:col10/1731773758627/Put/seqid=0 2024-11-16T16:16:00,311 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116d054bb65e0384a4096f2c7550649b372_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:00,313 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116d054bb65e0384a4096f2c7550649b372_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:00,313 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d054bb65e0384a4096f2c7550649b372_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:00,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742254_1430 (size=12779) 2024-11-16T16:16:00,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742255_1431 (size=4469) 2024-11-16T16:16:00,322 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#369 average throughput is 1.75 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:00,323 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/001088b0cafc4f029eea487b62aec99d is 175, key is test_row_0/A:col10/1731773758627/Put/seqid=0 2024-11-16T16:16:00,325 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/0defc587b0c44c37baa8c05bfaa7cfea as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0defc587b0c44c37baa8c05bfaa7cfea 2024-11-16T16:16:00,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742256_1432 (size=31733) 2024-11-16T16:16:00,335 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into 0defc587b0c44c37baa8c05bfaa7cfea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:00,335 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:00,335 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=13, startTime=1731773760302; duration=0sec 2024-11-16T16:16:00,335 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:00,335 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:16:00,335 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:00,336 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:00,336 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:16:00,336 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:00,336 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d80ca39ccf6546a594232c81067c49f6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.4 K 2024-11-16T16:16:00,337 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d80ca39ccf6546a594232c81067c49f6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731773754892 2024-11-16T16:16:00,337 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 77eeb970431e4961b65bd96fcb42a9e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773755608 2024-11-16T16:16:00,337 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4eb618dd784c64ba03f4ba9e3f16a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:00,343 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:00,344 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/23ba0fc9bea94fda9eb11b6e9b846298 is 50, key is test_row_0/C:col10/1731773758627/Put/seqid=0 2024-11-16T16:16:00,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742257_1433 (size=12779) 2024-11-16T16:16:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:00,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-16T16:16:00,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:00,409 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:00,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:00,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168ffb6275d4d84ffcacd7712731ef8c33_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773758700/Put/seqid=0 2024-11-16T16:16:00,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742258_1434 (size=12454) 2024-11-16T16:16:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:00,732 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/001088b0cafc4f029eea487b62aec99d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d 2024-11-16T16:16:00,744 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into 001088b0cafc4f029eea487b62aec99d(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:00,744 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:00,744 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=13, startTime=1731773760302; duration=0sec 2024-11-16T16:16:00,745 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:00,745 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:16:00,753 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/23ba0fc9bea94fda9eb11b6e9b846298 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/23ba0fc9bea94fda9eb11b6e9b846298 2024-11-16T16:16:00,759 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into 23ba0fc9bea94fda9eb11b6e9b846298(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:00,759 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:00,759 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=13, startTime=1731773760303; duration=0sec 2024-11-16T16:16:00,759 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:00,759 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:16:00,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:00,821 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168ffb6275d4d84ffcacd7712731ef8c33_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168ffb6275d4d84ffcacd7712731ef8c33_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:00,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/23f848af1e4346378046f155c2300ce6, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:00,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/23f848af1e4346378046f155c2300ce6 is 175, key is test_row_0/A:col10/1731773758700/Put/seqid=0 2024-11-16T16:16:00,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742259_1435 (size=31255) 2024-11-16T16:16:00,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:00,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773760996, max timestamp=1731773761024Num uniques 859; 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773760996, max timestamp=1731773761024Num uniques 859; 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773760996, max timestamp=1731773761024Num uniques 859; 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:01,027 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:01,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:01,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773821146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,155 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:01,230 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=406, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/23f848af1e4346378046f155c2300ce6 2024-11-16T16:16:01,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1f9c8625c62a40119b22d71eb5956e1d is 50, key is test_row_0/B:col10/1731773758700/Put/seqid=0 2024-11-16T16:16:01,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742260_1436 (size=12301) 2024-11-16T16:16:01,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773821265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:01,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:01,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773821475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:01,644 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1f9c8625c62a40119b22d71eb5956e1d 2024-11-16T16:16:01,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01b05afe2a9d4725a21d5499b1accbe4 is 50, key is test_row_0/C:col10/1731773758700/Put/seqid=0 2024-11-16T16:16:01,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742261_1437 (size=12301) 2024-11-16T16:16:01,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773821785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:01,791 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:02,065 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01b05afe2a9d4725a21d5499b1accbe4 2024-11-16T16:16:02,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/23f848af1e4346378046f155c2300ce6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6 2024-11-16T16:16:02,071 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6, entries=150, sequenceid=406, filesize=30.5 K 2024-11-16T16:16:02,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/1f9c8625c62a40119b22d71eb5956e1d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d 2024-11-16T16:16:02,078 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d, entries=150, sequenceid=406, filesize=12.0 K 2024-11-16T16:16:02,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/01b05afe2a9d4725a21d5499b1accbe4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4 2024-11-16T16:16:02,084 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4, entries=150, sequenceid=406, filesize=12.0 K 2024-11-16T16:16:02,085 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 951a043b6294ef712b77fdf5f51695c3 in 1676ms, sequenceid=406, compaction requested=false 2024-11-16T16:16:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-16T16:16:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-16T16:16:02,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-16T16:16:02,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8280 sec 2024-11-16T16:16:02,089 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.8330 sec 2024-11-16T16:16:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:02,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:16:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:02,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:02,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:02,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165826a93e4394485ab94896d16995577c_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742262_1438 (size=14994) 2024-11-16T16:16:02,341 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:02,344 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165826a93e4394485ab94896d16995577c_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165826a93e4394485ab94896d16995577c_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:02,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/bebd3f98e513463c81cbacfff263fae2, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/bebd3f98e513463c81cbacfff263fae2 is 175, key is test_row_0/A:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742263_1439 (size=39949) 2024-11-16T16:16:02,354 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=445, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/bebd3f98e513463c81cbacfff263fae2 2024-11-16T16:16:02,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/25b9059bd9df40479124a0891adc465f is 50, key is test_row_0/B:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742264_1440 (size=12301) 2024-11-16T16:16:02,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/25b9059bd9df40479124a0891adc465f 2024-11-16T16:16:02,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773822381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:02,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/053809769ef5463095090a902f62df2e is 50, key is test_row_0/C:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-16T16:16:02,405 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:02,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-16T16:16:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:02,407 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:02,408 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:02,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:02,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742265_1441 (size=12301) 2024-11-16T16:16:02,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/053809769ef5463095090a902f62df2e 2024-11-16T16:16:02,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/bebd3f98e513463c81cbacfff263fae2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2 2024-11-16T16:16:02,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2, entries=200, sequenceid=445, filesize=39.0 K 2024-11-16T16:16:02,435 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-16T16:16:02,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/25b9059bd9df40479124a0891adc465f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f 2024-11-16T16:16:02,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T16:16:02,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/053809769ef5463095090a902f62df2e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e 2024-11-16T16:16:02,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e, entries=150, sequenceid=445, filesize=12.0 K 2024-11-16T16:16:02,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 951a043b6294ef712b77fdf5f51695c3 in 145ms, sequenceid=445, compaction requested=true 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:02,451 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:02,451 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:16:02,453 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,453 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,453 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0defc587b0c44c37baa8c05bfaa7cfea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.5 K 2024-11-16T16:16:02,453 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=100.5 K 2024-11-16T16:16:02,453 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2] 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0defc587b0c44c37baa8c05bfaa7cfea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:02,453 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001088b0cafc4f029eea487b62aec99d, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:02,454 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23f848af1e4346378046f155c2300ce6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731773758676 2024-11-16T16:16:02,454 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f9c8625c62a40119b22d71eb5956e1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731773758676 2024-11-16T16:16:02,454 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting bebd3f98e513463c81cbacfff263fae2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:02,454 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 25b9059bd9df40479124a0891adc465f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:02,460 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:02,461 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/c9327a24fa77440a98d308d00c1b13a7 is 50, key is test_row_0/B:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,466 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,482 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111673ff090cdd6c41b9a94a2589ae63dc48_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,484 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111673ff090cdd6c41b9a94a2589ae63dc48_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,484 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111673ff090cdd6c41b9a94a2589ae63dc48_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742266_1442 (size=12881) 2024-11-16T16:16:02,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:02,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:02,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:02,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742267_1443 (size=4469) 2024-11-16T16:16:02,527 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#378 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:02,528 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/cf5388a225e847baa0eed3d439c5b0bf is 175, key is test_row_0/A:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c89c430638684e47b7ab845ea4ef7297_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773762366/Put/seqid=0 2024-11-16T16:16:02,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742268_1444 (size=31835) 2024-11-16T16:16:02,594 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/cf5388a225e847baa0eed3d439c5b0bf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf 2024-11-16T16:16:02,600 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into cf5388a225e847baa0eed3d439c5b0bf(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:02,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:02,600 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=13, startTime=1731773762451; duration=0sec 2024-11-16T16:16:02,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:02,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:16:02,600 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:02,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:02,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:16:02,603 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,603 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/23ba0fc9bea94fda9eb11b6e9b846298, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=36.5 K 2024-11-16T16:16:02,603 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ba0fc9bea94fda9eb11b6e9b846298, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=394, earliestPutTs=1731773757909 2024-11-16T16:16:02,604 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01b05afe2a9d4725a21d5499b1accbe4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731773758676 2024-11-16T16:16:02,604 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 053809769ef5463095090a902f62df2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:02,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742269_1445 (size=12454) 2024-11-16T16:16:02,629 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:02,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c89c430638684e47b7ab845ea4ef7297_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c89c430638684e47b7ab845ea4ef7297_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:02,634 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/7bc4f3a8dd924380bf8267aaeb535f93, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:02,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/7bc4f3a8dd924380bf8267aaeb535f93 is 175, key is test_row_0/A:col10/1731773762366/Put/seqid=0 2024-11-16T16:16:02,639 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:02,640 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/3d8dc2ff2e994cbf945c3c3a0ab566b8 is 50, key is test_row_0/C:col10/1731773762304/Put/seqid=0 2024-11-16T16:16:02,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742270_1446 (size=31255) 2024-11-16T16:16:02,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/7bc4f3a8dd924380bf8267aaeb535f93 2024-11-16T16:16:02,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:02,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742271_1447 (size=12881) 2024-11-16T16:16:02,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/5dd366a1c659421a8d5cc006e279225f is 50, key is test_row_0/B:col10/1731773762366/Put/seqid=0 2024-11-16T16:16:02,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:02,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742272_1448 (size=12301) 2024-11-16T16:16:02,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/5dd366a1c659421a8d5cc006e279225f 2024-11-16T16:16:02,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/2a82f3c7c3a0429bb09b98a693b6ee0a is 50, key is test_row_0/C:col10/1731773762366/Put/seqid=0 2024-11-16T16:16:02,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742273_1449 (size=12301) 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773762728, max timestamp=1731773762816Num uniques 859; 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773762728, max timestamp=1731773762816Num uniques 859; 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773762728, max timestamp=1731773762816Num uniques 859; 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:02,819 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:02,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:02,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:02,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:02,911 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/c9327a24fa77440a98d308d00c1b13a7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/c9327a24fa77440a98d308d00c1b13a7 2024-11-16T16:16:02,917 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into c9327a24fa77440a98d308d00c1b13a7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:02,917 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:02,917 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=13, startTime=1731773762451; duration=0sec 2024-11-16T16:16:02,917 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:02,917 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:16:03,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:03,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773823006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:03,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:03,023 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:03,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773823125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:03,133 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/3d8dc2ff2e994cbf945c3c3a0ab566b8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d8dc2ff2e994cbf945c3c3a0ab566b8 2024-11-16T16:16:03,138 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into 3d8dc2ff2e994cbf945c3c3a0ab566b8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:03,138 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:03,138 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=13, startTime=1731773762451; duration=0sec 2024-11-16T16:16:03,138 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:03,138 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:16:03,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:03,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:03,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/2a82f3c7c3a0429bb09b98a693b6ee0a 2024-11-16T16:16:03,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/7bc4f3a8dd924380bf8267aaeb535f93 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93 2024-11-16T16:16:03,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93, entries=150, sequenceid=456, filesize=30.5 K 2024-11-16T16:16:03,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/5dd366a1c659421a8d5cc006e279225f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f 2024-11-16T16:16:03,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T16:16:03,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/2a82f3c7c3a0429bb09b98a693b6ee0a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a 2024-11-16T16:16:03,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a, entries=150, sequenceid=456, filesize=12.0 K 2024-11-16T16:16:03,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 951a043b6294ef712b77fdf5f51695c3 in 724ms, sequenceid=456, compaction requested=false 2024-11-16T16:16:03,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:03,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-16T16:16:03,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:03,333 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:03,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:03,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165a3b9f6d656b4f91b8af5990a150f66e_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773762847/Put/seqid=0 2024-11-16T16:16:03,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742274_1450 (size=12454) 2024-11-16T16:16:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773823429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:03,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:03,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773823557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773823786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:03,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:03,826 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411165a3b9f6d656b4f91b8af5990a150f66e_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a3b9f6d656b4f91b8af5990a150f66e_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:03,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/aa905174e6e2468493006c9c79486857, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:03,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/aa905174e6e2468493006c9c79486857 is 175, key is test_row_0/A:col10/1731773762847/Put/seqid=0 2024-11-16T16:16:03,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742275_1451 (size=31255) 2024-11-16T16:16:04,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:04,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773824117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:04,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:04,279 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=495, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/aa905174e6e2468493006c9c79486857 2024-11-16T16:16:04,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9cb9278988624af0886b00f8e5c7ebc5 is 50, key is test_row_0/B:col10/1731773762847/Put/seqid=0 2024-11-16T16:16:04,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742276_1452 (size=12301) 2024-11-16T16:16:04,360 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9cb9278988624af0886b00f8e5c7ebc5 2024-11-16T16:16:04,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/562aa3e122e444678c061b9e8ddc9eda is 50, key is test_row_0/C:col10/1731773762847/Put/seqid=0 2024-11-16T16:16:04,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742277_1453 (size=12301) 2024-11-16T16:16:04,439 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/562aa3e122e444678c061b9e8ddc9eda 2024-11-16T16:16:04,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/aa905174e6e2468493006c9c79486857 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857 2024-11-16T16:16:04,452 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857, entries=150, sequenceid=495, filesize=30.5 K 2024-11-16T16:16:04,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/9cb9278988624af0886b00f8e5c7ebc5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5 2024-11-16T16:16:04,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,458 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5, entries=150, sequenceid=495, filesize=12.0 K 2024-11-16T16:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/562aa3e122e444678c061b9e8ddc9eda as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda 2024-11-16T16:16:04,464 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda, entries=150, sequenceid=495, filesize=12.0 K 2024-11-16T16:16:04,465 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 951a043b6294ef712b77fdf5f51695c3 in 1132ms, sequenceid=495, compaction requested=true 2024-11-16T16:16:04,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:04,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:04,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-16T16:16:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-16T16:16:04,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-16T16:16:04,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0580 sec 2024-11-16T16:16:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0730 sec 2024-11-16T16:16:04,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-16T16:16:04,555 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-16T16:16:04,562 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:04,563 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:04,563 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:04,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:04,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:04,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:04,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:04,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c21c0ee28478430cbbdf2b8f55275c56_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:04,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:04,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:04,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742278_1454 (size=12454) 2024-11-16T16:16:04,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:04,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:04,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:04,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:05,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:05,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773825075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:05,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:05,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c21c0ee28478430cbbdf2b8f55275c56_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c21c0ee28478430cbbdf2b8f55275c56_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:05,184 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/19bac4bd9d964d789587005852672942, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:05,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/19bac4bd9d964d789587005852672942 is 175, key is test_row_0/A:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:05,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773825195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:05,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742279_1455 (size=31255) 2024-11-16T16:16:05,318 INFO [Thread-1695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1695. 2024-11-16T16:16:05,318 DEBUG [Thread-1695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:05,318 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,318 DEBUG [Thread-1695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,318 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:05,324 INFO [Thread-1697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1697. 2024-11-16T16:16:05,324 DEBUG [Thread-1697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:05,324 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,324 DEBUG [Thread-1697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:05,325 INFO [Thread-1699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1699. 2024-11-16T16:16:05,325 DEBUG [Thread-1699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:05,325 DEBUG [Thread-1699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,325 DEBUG [Thread-1699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:05,334 INFO [Thread-1693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1693. 2024-11-16T16:16:05,334 DEBUG [Thread-1693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:05,334 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,334 DEBUG [Thread-1693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:05,336 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-11-16T16:16:05,336 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:05,336 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,336 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:05,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:05,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 287 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773825435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:05,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:05,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,650 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=507, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/19bac4bd9d964d789587005852672942 2024-11-16T16:16:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/413117ebb7784f01bca0b3a76ed86ad8 is 50, key is test_row_0/B:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:05,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742280_1456 (size=12301) 2024-11-16T16:16:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:05,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773825745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:05,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:05,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:05,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:05,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/413117ebb7784f01bca0b3a76ed86ad8 2024-11-16T16:16:06,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/d0528dde14bc435180f2cd8a0181fe94 is 50, key is test_row_0/C:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:06,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742281_1457 (size=12301) 2024-11-16T16:16:06,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:06,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:06,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,110 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45412 deadline: 1731773826256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:06,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:06,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:06,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:06,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:06,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:06,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:06,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:06,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/d0528dde14bc435180f2cd8a0181fe94 2024-11-16T16:16:06,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/19bac4bd9d964d789587005852672942 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942 2024-11-16T16:16:06,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942, entries=150, sequenceid=507, filesize=30.5 K 2024-11-16T16:16:06,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/413117ebb7784f01bca0b3a76ed86ad8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8 2024-11-16T16:16:06,492 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8, entries=150, sequenceid=507, filesize=12.0 K 2024-11-16T16:16:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/d0528dde14bc435180f2cd8a0181fe94 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94 2024-11-16T16:16:06,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94, entries=150, sequenceid=507, filesize=12.0 K 2024-11-16T16:16:06,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 951a043b6294ef712b77fdf5f51695c3 in 1816ms, sequenceid=507, compaction requested=true 2024-11-16T16:16:06,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:06,497 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:06,497 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 951a043b6294ef712b77fdf5f51695c3:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:06,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:06,497 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:06,497 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/B is initiating minor compaction (all files) 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/A is initiating minor compaction (all files) 2024-11-16T16:16:06,498 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/B in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,498 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/A in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,498 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/c9327a24fa77440a98d308d00c1b13a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=48.6 K 2024-11-16T16:16:06,498 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=122.7 K 2024-11-16T16:16:06,498 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942] 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c9327a24fa77440a98d308d00c1b13a7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf5388a225e847baa0eed3d439c5b0bf, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 5dd366a1c659421a8d5cc006e279225f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773762329 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bc4f3a8dd924380bf8267aaeb535f93, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773762329 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb9278988624af0886b00f8e5c7ebc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1731773762847 2024-11-16T16:16:06,498 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa905174e6e2468493006c9c79486857, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1731773762847 2024-11-16T16:16:06,499 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 413117ebb7784f01bca0b3a76ed86ad8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773763407 2024-11-16T16:16:06,499 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19bac4bd9d964d789587005852672942, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773763407 2024-11-16T16:16:06,508 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:06,508 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#B#compaction#389 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:06,509 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/72168952444e4b6aae4db8ddceef19b3 is 50, key is test_row_0/B:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:06,509 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116792da07c5ccc43519d2a5c7a79f1830e_951a043b6294ef712b77fdf5f51695c3 store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:06,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742282_1458 (size=13017) 2024-11-16T16:16:06,531 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116792da07c5ccc43519d2a5c7a79f1830e_951a043b6294ef712b77fdf5f51695c3, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:06,531 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116792da07c5ccc43519d2a5c7a79f1830e_951a043b6294ef712b77fdf5f51695c3 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:06,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742283_1459 (size=4469) 2024-11-16T16:16:06,545 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#A#compaction#390 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:06,546 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/f20d739bf4a34ca6a1a656847748dc3e is 175, key is test_row_0/A:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:06,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742284_1460 (size=31971) 2024-11-16T16:16:06,559 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/f20d739bf4a34ca6a1a656847748dc3e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/f20d739bf4a34ca6a1a656847748dc3e 2024-11-16T16:16:06,565 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/A of 951a043b6294ef712b77fdf5f51695c3 into f20d739bf4a34ca6a1a656847748dc3e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:06,565 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:06,565 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/A, priority=12, startTime=1731773766496; duration=0sec 2024-11-16T16:16:06,565 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:06,565 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:A 2024-11-16T16:16:06,565 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:06,567 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:06,567 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 951a043b6294ef712b77fdf5f51695c3/C is initiating minor compaction (all files) 2024-11-16T16:16:06,568 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 951a043b6294ef712b77fdf5f51695c3/C in TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,568 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d8dc2ff2e994cbf945c3c3a0ab566b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp, totalSize=48.6 K 2024-11-16T16:16:06,568 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d8dc2ff2e994cbf945c3c3a0ab566b8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731773761103 2024-11-16T16:16:06,569 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a82f3c7c3a0429bb09b98a693b6ee0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731773762329 2024-11-16T16:16:06,569 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 562aa3e122e444678c061b9e8ddc9eda, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=495, earliestPutTs=1731773762847 2024-11-16T16:16:06,569 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0528dde14bc435180f2cd8a0181fe94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731773763407 2024-11-16T16:16:06,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-16T16:16:06,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:06,571 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:06,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:06,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111680f5c661cfbe4ebeaa0e6f731b71c3b1_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773764865/Put/seqid=0 2024-11-16T16:16:06,593 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 951a043b6294ef712b77fdf5f51695c3#C#compaction#391 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:06,593 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/26f48a5d423d4ab9bce67e6338d574dc is 50, key is test_row_0/C:col10/1731773763427/Put/seqid=0 2024-11-16T16:16:06,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742285_1461 (size=12454) 2024-11-16T16:16:06,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742286_1462 (size=13017) 2024-11-16T16:16:06,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:06,918 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/72168952444e4b6aae4db8ddceef19b3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/72168952444e4b6aae4db8ddceef19b3 2024-11-16T16:16:06,923 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/B of 951a043b6294ef712b77fdf5f51695c3 into 72168952444e4b6aae4db8ddceef19b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:06,923 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:06,923 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/B, priority=12, startTime=1731773766497; duration=0sec 2024-11-16T16:16:06,923 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:06,923 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:B 2024-11-16T16:16:07,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:07,021 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111680f5c661cfbe4ebeaa0e6f731b71c3b1_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111680f5c661cfbe4ebeaa0e6f731b71c3b1_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:07,021 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/26f48a5d423d4ab9bce67e6338d574dc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/26f48a5d423d4ab9bce67e6338d574dc 2024-11-16T16:16:07,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/519e9cbb7866455a942676b73a4ac7ee, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:07,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/519e9cbb7866455a942676b73a4ac7ee is 175, key is test_row_0/A:col10/1731773764865/Put/seqid=0 2024-11-16T16:16:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742287_1463 (size=31255) 2024-11-16T16:16:07,027 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 951a043b6294ef712b77fdf5f51695c3/C of 951a043b6294ef712b77fdf5f51695c3 into 26f48a5d423d4ab9bce67e6338d574dc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:07,027 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:07,027 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., storeName=951a043b6294ef712b77fdf5f51695c3/C, priority=12, startTime=1731773766497; duration=0sec 2024-11-16T16:16:07,027 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:07,027 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 951a043b6294ef712b77fdf5f51695c3:C 2024-11-16T16:16:07,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:07,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. as already flushing 2024-11-16T16:16:07,078 INFO [Thread-1688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1688. 2024-11-16T16:16:07,078 DEBUG [Thread-1688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:07,078 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:07,078 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:07,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:07,268 INFO [Thread-1682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1682. 2024-11-16T16:16:07,268 DEBUG [Thread-1682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:07,268 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:07,268 DEBUG [Thread-1682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:07,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:07,428 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=528, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/519e9cbb7866455a942676b73a4ac7ee 2024-11-16T16:16:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/709c4ad951f848abbe4e606241fefde7 is 50, key is test_row_0/B:col10/1731773764865/Put/seqid=0 2024-11-16T16:16:07,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742288_1464 (size=12301) 2024-11-16T16:16:07,844 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/709c4ad951f848abbe4e606241fefde7 2024-11-16T16:16:07,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a8cc25b90aa943f091aaf67b971588ff is 50, key is test_row_0/C:col10/1731773764865/Put/seqid=0 2024-11-16T16:16:07,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742289_1465 (size=12301) 2024-11-16T16:16:07,877 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a8cc25b90aa943f091aaf67b971588ff 2024-11-16T16:16:07,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/519e9cbb7866455a942676b73a4ac7ee as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/519e9cbb7866455a942676b73a4ac7ee 2024-11-16T16:16:07,889 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/519e9cbb7866455a942676b73a4ac7ee, entries=150, sequenceid=528, filesize=30.5 K 2024-11-16T16:16:07,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/709c4ad951f848abbe4e606241fefde7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/709c4ad951f848abbe4e606241fefde7 2024-11-16T16:16:07,894 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/709c4ad951f848abbe4e606241fefde7, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T16:16:07,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/a8cc25b90aa943f091aaf67b971588ff as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a8cc25b90aa943f091aaf67b971588ff 2024-11-16T16:16:07,900 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a8cc25b90aa943f091aaf67b971588ff, entries=150, sequenceid=528, filesize=12.0 K 2024-11-16T16:16:07,901 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for 951a043b6294ef712b77fdf5f51695c3 in 1330ms, sequenceid=528, compaction requested=false 2024-11-16T16:16:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 951a043b6294ef712b77fdf5f51695c3: 2024-11-16T16:16:07,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:07,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-16T16:16:07,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-16T16:16:07,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-16T16:16:07,905 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3400 sec 2024-11-16T16:16:07,907 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 3.3490 sec 2024-11-16T16:16:08,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-16T16:16:08,725 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:09,187 INFO [Thread-1684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1684. 2024-11-16T16:16:09,187 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-11-16T16:16:09,187 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:09,187 DEBUG [Thread-1684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:09,188 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,188 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,188 DEBUG [Thread-1684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,188 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:09,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:09,259 INFO [Thread-1686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1686. 2024-11-16T16:16:09,260 DEBUG [Thread-1686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:09,260 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,260 DEBUG [Thread-1686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,260 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:09,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 237 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 994 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2978 rows 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 983 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2942 rows 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 992 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2975 rows 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 986 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2955 rows 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 993 2024-11-16T16:16:09,261 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2978 rows 2024-11-16T16:16:09,262 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:16:09,262 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:16:09,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:09,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:09,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:16:09,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:09,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:09,270 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773769269"}]},"ts":"1731773769269"} 2024-11-16T16:16:09,271 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:16:09,271 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:16:09,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:16:09,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, UNASSIGN}] 2024-11-16T16:16:09,273 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, UNASSIGN 2024-11-16T16:16:09,274 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:09,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, UNASSIGN because future has completed 2024-11-16T16:16:09,275 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:16:09,275 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:09,427 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:09,427 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:16:09,427 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 951a043b6294ef712b77fdf5f51695c3, disabling compactions & flushes 2024-11-16T16:16:09,427 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:09,427 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:09,427 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. after waiting 0 ms 2024-11-16T16:16:09,427 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:09,428 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 951a043b6294ef712b77fdf5f51695c3 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=A 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=B 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 951a043b6294ef712b77fdf5f51695c3, store=C 2024-11-16T16:16:09,428 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:09,437 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161af5369c509043fd90a0398a6d764fcf_951a043b6294ef712b77fdf5f51695c3 is 50, key is test_row_0/A:col10/1731773769256/Put/seqid=0 2024-11-16T16:16:09,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742290_1466 (size=12454) 2024-11-16T16:16:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:09,843 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:09,847 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161af5369c509043fd90a0398a6d764fcf_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161af5369c509043fd90a0398a6d764fcf_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:09,848 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/143b6349468c461b84f883a34ba5c74f, store: [table=TestAcidGuarantees family=A region=951a043b6294ef712b77fdf5f51695c3] 2024-11-16T16:16:09,848 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/143b6349468c461b84f883a34ba5c74f is 175, key is test_row_0/A:col10/1731773769256/Put/seqid=0 2024-11-16T16:16:09,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742291_1467 (size=31255) 2024-11-16T16:16:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:10,254 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=538, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/143b6349468c461b84f883a34ba5c74f 2024-11-16T16:16:10,266 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/d12e7c81b19e47f6a185a03f5edfd5e4 is 50, key is test_row_0/B:col10/1731773769256/Put/seqid=0 2024-11-16T16:16:10,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742292_1468 (size=12301) 2024-11-16T16:16:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:10,671 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/d12e7c81b19e47f6a185a03f5edfd5e4 2024-11-16T16:16:10,685 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/c48a44f5e39444519408effb8d9df8c8 is 50, key is test_row_0/C:col10/1731773769256/Put/seqid=0 2024-11-16T16:16:10,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742293_1469 (size=12301) 2024-11-16T16:16:10,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:16:11,090 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/c48a44f5e39444519408effb8d9df8c8 2024-11-16T16:16:11,097 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/A/143b6349468c461b84f883a34ba5c74f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/143b6349468c461b84f883a34ba5c74f 2024-11-16T16:16:11,101 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/143b6349468c461b84f883a34ba5c74f, entries=150, sequenceid=538, filesize=30.5 K 2024-11-16T16:16:11,101 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/B/d12e7c81b19e47f6a185a03f5edfd5e4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/d12e7c81b19e47f6a185a03f5edfd5e4 2024-11-16T16:16:11,104 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/d12e7c81b19e47f6a185a03f5edfd5e4, entries=150, sequenceid=538, filesize=12.0 K 2024-11-16T16:16:11,105 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/.tmp/C/c48a44f5e39444519408effb8d9df8c8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/c48a44f5e39444519408effb8d9df8c8 2024-11-16T16:16:11,107 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/c48a44f5e39444519408effb8d9df8c8, entries=150, sequenceid=538, filesize=12.0 K 2024-11-16T16:16:11,108 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 951a043b6294ef712b77fdf5f51695c3 in 1681ms, sequenceid=538, compaction requested=true 2024-11-16T16:16:11,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942] to archive 2024-11-16T16:16:11,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:11,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4a1ccc01d2234a31963c4867027798a9 2024-11-16T16:16:11,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c76890d8f4fa4e899b608a3f65f37b06 2024-11-16T16:16:11,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/719ce248f79e4c22be67e06fd82b9c8f 2024-11-16T16:16:11,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/d5b1b55f9a1646a3ab30f1c432727906 2024-11-16T16:16:11,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/37de7596c5044ddea2bc79154492e2e6 2024-11-16T16:16:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/417e918f76ae4a32aa0ead1977130724 2024-11-16T16:16:11,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/3017a25fd2214d14a5b7fea25fda0e59 2024-11-16T16:16:11,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/89e2695271a84548878feeb796c2bb61 2024-11-16T16:16:11,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/df39f14cca2d4188968a4d635ec31aac 2024-11-16T16:16:11,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/c3f0825c76c545e08af63657dab3e887 2024-11-16T16:16:11,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/99d120044e04434687ecc35d91caa05d 2024-11-16T16:16:11,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/527435370e6a495f949fb8f77639ddc4 2024-11-16T16:16:11,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/4530751bbc51490da4a89f238e18a53e 2024-11-16T16:16:11,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/45dc5796b66a4af7b97ad630259efe9c 2024-11-16T16:16:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/001088b0cafc4f029eea487b62aec99d 2024-11-16T16:16:11,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/23f848af1e4346378046f155c2300ce6 2024-11-16T16:16:11,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/bebd3f98e513463c81cbacfff263fae2 2024-11-16T16:16:11,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/cf5388a225e847baa0eed3d439c5b0bf 2024-11-16T16:16:11,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/7bc4f3a8dd924380bf8267aaeb535f93 2024-11-16T16:16:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/aa905174e6e2468493006c9c79486857 2024-11-16T16:16:11,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/19bac4bd9d964d789587005852672942 2024-11-16T16:16:11,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/e04a8a6328e647b09ec08b5d85baa2c7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/acf784a609a044e7be7cf074d86af233, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/93837b9623a34c3c9724f4e8ad8f6e19, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0defc587b0c44c37baa8c05bfaa7cfea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/c9327a24fa77440a98d308d00c1b13a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8] to archive 2024-11-16T16:16:11,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:11,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/98bb4f4877d047c6876cba5141e2319b 2024-11-16T16:16:11,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1fbafdba23154528bd02648bcd812b02 2024-11-16T16:16:11,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/b4af1f6e583b42f68375cf95b76d0edf 2024-11-16T16:16:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/e04a8a6328e647b09ec08b5d85baa2c7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/e04a8a6328e647b09ec08b5d85baa2c7 2024-11-16T16:16:11,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0d336eea7f9b4eb6b946d33f63c4d5ab 2024-11-16T16:16:11,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/dfc4b1ec569a43e98ae57159e8903307 2024-11-16T16:16:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/acf784a609a044e7be7cf074d86af233 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/acf784a609a044e7be7cf074d86af233 2024-11-16T16:16:11,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/7a36246b60ca4abe87939911a97b7ab3 2024-11-16T16:16:11,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/47c0b7a0904d43b2ab781504fba74db3 2024-11-16T16:16:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/55cdd62daf87460294477696cf7e4a98 2024-11-16T16:16:11,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/93837b9623a34c3c9724f4e8ad8f6e19 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/93837b9623a34c3c9724f4e8ad8f6e19 2024-11-16T16:16:11,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9f5f597f91a74394a946eb8b146ed8fe 2024-11-16T16:16:11,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/57d84e6c5f634bb8a8da83842576e164 2024-11-16T16:16:11,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0defc587b0c44c37baa8c05bfaa7cfea to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/0defc587b0c44c37baa8c05bfaa7cfea 2024-11-16T16:16:11,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/047c910dfa194416a3d442ccacd16cab 2024-11-16T16:16:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/1f9c8625c62a40119b22d71eb5956e1d 2024-11-16T16:16:11,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/c9327a24fa77440a98d308d00c1b13a7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/c9327a24fa77440a98d308d00c1b13a7 2024-11-16T16:16:11,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/25b9059bd9df40479124a0891adc465f 2024-11-16T16:16:11,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/5dd366a1c659421a8d5cc006e279225f 2024-11-16T16:16:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/9cb9278988624af0886b00f8e5c7ebc5 2024-11-16T16:16:11,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/413117ebb7784f01bca0b3a76ed86ad8 2024-11-16T16:16:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/ecdfc727d828428f98c009e5a4c6cfc7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8140772a5c934c59a8caa8118fdc1735, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d80ca39ccf6546a594232c81067c49f6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/23ba0fc9bea94fda9eb11b6e9b846298, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d8dc2ff2e994cbf945c3c3a0ab566b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94] to archive 2024-11-16T16:16:11,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:11,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/21b42c4bb3dd4f80b85a53f3b13b4e39 2024-11-16T16:16:11,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d6d9c937ea64b1692d27dbf3947c64d 2024-11-16T16:16:11,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/df3ce15e31ae4f01addaf8ce5bd2c21a 2024-11-16T16:16:11,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/ecdfc727d828428f98c009e5a4c6cfc7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/ecdfc727d828428f98c009e5a4c6cfc7 2024-11-16T16:16:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f22901fdd46047cb9f256eee5edccaa8 2024-11-16T16:16:11,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a3790a0860eb4e848833bc21dc079bfc 2024-11-16T16:16:11,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8140772a5c934c59a8caa8118fdc1735 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8140772a5c934c59a8caa8118fdc1735 2024-11-16T16:16:11,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01e49c7bf08f46fba2b4dbfc8b6ad8f5 2024-11-16T16:16:11,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/fbc896a9a6bd422480189c618fc465fe 2024-11-16T16:16:11,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/f7d71dfe36904b359ea85c55b1b01920 2024-11-16T16:16:11,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d80ca39ccf6546a594232c81067c49f6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d80ca39ccf6546a594232c81067c49f6 2024-11-16T16:16:11,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/113701ba5c56400f8a555df24b80c468 2024-11-16T16:16:11,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/77eeb970431e4961b65bd96fcb42a9e3 2024-11-16T16:16:11,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/23ba0fc9bea94fda9eb11b6e9b846298 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/23ba0fc9bea94fda9eb11b6e9b846298 2024-11-16T16:16:11,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/8e4eb618dd784c64ba03f4ba9e3f16a3 2024-11-16T16:16:11,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/01b05afe2a9d4725a21d5499b1accbe4 2024-11-16T16:16:11,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d8dc2ff2e994cbf945c3c3a0ab566b8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/3d8dc2ff2e994cbf945c3c3a0ab566b8 2024-11-16T16:16:11,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/053809769ef5463095090a902f62df2e 2024-11-16T16:16:11,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/2a82f3c7c3a0429bb09b98a693b6ee0a 2024-11-16T16:16:11,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/562aa3e122e444678c061b9e8ddc9eda 2024-11-16T16:16:11,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/d0528dde14bc435180f2cd8a0181fe94 2024-11-16T16:16:11,171 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits/541.seqid, newMaxSeqId=541, maxSeqId=4 2024-11-16T16:16:11,171 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. 2024-11-16T16:16:11,171 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 951a043b6294ef712b77fdf5f51695c3: Waiting for close lock at 1731773769427Running coprocessor pre-close hooks at 1731773769427Disabling compacts and flushes for region at 1731773769427Disabling writes for close at 1731773769427Obtaining lock to block concurrent updates at 1731773769428 (+1 ms)Preparing flush snapshotting stores in 951a043b6294ef712b77fdf5f51695c3 at 1731773769428Finished memstore snapshotting TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731773769428Flushing stores of TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3. at 1731773769429 (+1 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/A: creating writer at 1731773769429Flushing mob file 951a043b6294ef712b77fdf5f51695c3/A: appending metadata at 1731773769437 (+8 ms)Flushing mob file 951a043b6294ef712b77fdf5f51695c3/A: closing flushed file at 1731773769437Flushing 951a043b6294ef712b77fdf5f51695c3/A: appending metadata at 1731773769848 (+411 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/A: closing flushed file at 1731773769848Flushing 951a043b6294ef712b77fdf5f51695c3/B: creating writer at 1731773770263 (+415 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/B: appending metadata at 1731773770266 (+3 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/B: closing flushed file at 1731773770266Flushing 951a043b6294ef712b77fdf5f51695c3/C: creating writer at 1731773770681 (+415 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/C: appending metadata at 1731773770685 (+4 ms)Flushing 951a043b6294ef712b77fdf5f51695c3/C: closing flushed file at 1731773770685Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b69c22c: reopening flushed file at 1731773771096 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@631a37f0: reopening flushed file at 1731773771101 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@698203a0: reopening flushed file at 1731773771104 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 951a043b6294ef712b77fdf5f51695c3 in 1681ms, sequenceid=538, compaction requested=true at 1731773771108 (+4 ms)Writing region close event to WAL at 1731773771168 (+60 ms)Running coprocessor post-close hooks at 1731773771171 (+3 ms)Closed at 1731773771171 2024-11-16T16:16:11,173 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,174 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=951a043b6294ef712b77fdf5f51695c3, regionState=CLOSED 2024-11-16T16:16:11,175 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:11,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-16T16:16:11,176 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 951a043b6294ef712b77fdf5f51695c3, server=303fe01965e8,37355,1731773652566 in 1.9000 sec 2024-11-16T16:16:11,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-16T16:16:11,178 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=951a043b6294ef712b77fdf5f51695c3, UNASSIGN in 1.9030 sec 2024-11-16T16:16:11,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-16T16:16:11,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9070 sec 2024-11-16T16:16:11,181 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773771180"}]},"ts":"1731773771180"} 2024-11-16T16:16:11,182 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:16:11,182 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:16:11,184 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9190 sec 2024-11-16T16:16:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-16T16:16:11,426 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:11,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:16:11,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,428 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,428 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T16:16:11,430 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,432 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits] 2024-11-16T16:16:11,434 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/143b6349468c461b84f883a34ba5c74f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/143b6349468c461b84f883a34ba5c74f 2024-11-16T16:16:11,435 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/519e9cbb7866455a942676b73a4ac7ee to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/519e9cbb7866455a942676b73a4ac7ee 2024-11-16T16:16:11,436 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/f20d739bf4a34ca6a1a656847748dc3e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/A/f20d739bf4a34ca6a1a656847748dc3e 2024-11-16T16:16:11,437 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/709c4ad951f848abbe4e606241fefde7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/709c4ad951f848abbe4e606241fefde7 2024-11-16T16:16:11,438 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/72168952444e4b6aae4db8ddceef19b3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/72168952444e4b6aae4db8ddceef19b3 2024-11-16T16:16:11,439 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/d12e7c81b19e47f6a185a03f5edfd5e4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/B/d12e7c81b19e47f6a185a03f5edfd5e4 2024-11-16T16:16:11,440 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/26f48a5d423d4ab9bce67e6338d574dc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/26f48a5d423d4ab9bce67e6338d574dc 2024-11-16T16:16:11,441 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a8cc25b90aa943f091aaf67b971588ff to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/a8cc25b90aa943f091aaf67b971588ff 2024-11-16T16:16:11,441 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/c48a44f5e39444519408effb8d9df8c8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/C/c48a44f5e39444519408effb8d9df8c8 2024-11-16T16:16:11,443 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits/541.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3/recovered.edits/541.seqid 2024-11-16T16:16:11,444 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,444 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:16:11,444 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:16:11,445 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T16:16:11,447 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160bacf44e7d634ac2bb47bf8cef32281d_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160bacf44e7d634ac2bb47bf8cef32281d_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,447 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161af5369c509043fd90a0398a6d764fcf_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161af5369c509043fd90a0398a6d764fcf_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,448 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bbdb7649c1f4dadbf7fe569b5ec394c_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161bbdb7649c1f4dadbf7fe569b5ec394c_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,449 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111639d017a8dd31432aab43028386a0f447_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111639d017a8dd31432aab43028386a0f447_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,449 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c9bd9d41040459cbb14b525015d69c7_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164c9bd9d41040459cbb14b525015d69c7_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,450 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165826a93e4394485ab94896d16995577c_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165826a93e4394485ab94896d16995577c_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,451 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a3b9f6d656b4f91b8af5990a150f66e_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165a3b9f6d656b4f91b8af5990a150f66e_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,452 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fd6d6b7e87c468a92e1c867a01fe660_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411165fd6d6b7e87c468a92e1c867a01fe660_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,453 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111680f5c661cfbe4ebeaa0e6f731b71c3b1_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111680f5c661cfbe4ebeaa0e6f731b71c3b1_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,453 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168ffb6275d4d84ffcacd7712731ef8c33_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168ffb6275d4d84ffcacd7712731ef8c33_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,454 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169757046992ab4485a27aaaea686f3d14_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411169757046992ab4485a27aaaea686f3d14_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,455 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae2b8ec72ee8486899c1009a4d496df3_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ae2b8ec72ee8486899c1009a4d496df3_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,455 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ba31850daa72492a9004ddeebc3c0953_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ba31850daa72492a9004ddeebc3c0953_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,456 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c21c0ee28478430cbbdf2b8f55275c56_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c21c0ee28478430cbbdf2b8f55275c56_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,457 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c89c430638684e47b7ab845ea4ef7297_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c89c430638684e47b7ab845ea4ef7297_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,457 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d1947738dc9b446cbe68c66809674d80_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d1947738dc9b446cbe68c66809674d80_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,458 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4da6854114144718a65070a391d83aa_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e4da6854114144718a65070a391d83aa_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,458 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5a25931bd5e4452b7ced5fa3917ee32_951a043b6294ef712b77fdf5f51695c3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116e5a25931bd5e4452b7ced5fa3917ee32_951a043b6294ef712b77fdf5f51695c3 2024-11-16T16:16:11,459 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:16:11,460 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,462 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:16:11,463 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:16:11,464 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,464 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:16:11,464 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773771464"}]},"ts":"9223372036854775807"} 2024-11-16T16:16:11,466 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:16:11,466 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 951a043b6294ef712b77fdf5f51695c3, NAME => 'TestAcidGuarantees,,1731773742173.951a043b6294ef712b77fdf5f51695c3.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:16:11,466 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:16:11,466 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773771466"}]},"ts":"9223372036854775807"} 2024-11-16T16:16:11,468 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:16:11,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,469 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-16T16:16:11,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-16T16:16:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:16:11,536 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:11,545 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 249) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-500171457_22 at /127.0.0.1:42200 [Waiting for operation #733] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-500171457_22 at /127.0.0.1:39134 [Waiting for operation #664] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1459812294_22 at /127.0.0.1:39150 [Waiting for operation #703] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1459812294_22 at /127.0.0.1:39148 [Waiting for operation #690] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 506) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3826 (was 4430) 2024-11-16T16:16:11,554 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=3826 2024-11-16T16:16:11,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:16:11,555 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:16:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:11,557 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:16:11,557 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:11,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-16T16:16:11,558 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:16:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:11,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742294_1470 (size=1061) 2024-11-16T16:16:11,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:11,965 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:16:11,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742295_1471 (size=53) 2024-11-16T16:16:12,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:12,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:16:12,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e301819a7ac81b4179f3ff5e1e4d8bad, disabling compactions & flushes 2024-11-16T16:16:12,375 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. after waiting 0 ms 2024-11-16T16:16:12,375 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,375 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,376 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e301819a7ac81b4179f3ff5e1e4d8bad: Waiting for close lock at 1731773772375Disabling compacts and flushes for region at 1731773772375Disabling writes for close at 1731773772375Writing region close event to WAL at 1731773772375Closed at 1731773772375 2024-11-16T16:16:12,378 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:16:12,378 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773772378"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773772378"}]},"ts":"1731773772378"} 2024-11-16T16:16:12,381 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:16:12,382 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:16:12,383 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773772382"}]},"ts":"1731773772382"} 2024-11-16T16:16:12,385 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:16:12,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, ASSIGN}] 2024-11-16T16:16:12,387 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, ASSIGN 2024-11-16T16:16:12,387 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:16:12,538 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e301819a7ac81b4179f3ff5e1e4d8bad, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:12,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, ASSIGN because future has completed 2024-11-16T16:16:12,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:12,816 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,817 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:16:12,818 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,818 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:16:12,819 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,819 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,822 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,825 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:12,826 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e301819a7ac81b4179f3ff5e1e4d8bad columnFamilyName A 2024-11-16T16:16:12,826 DEBUG [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:12,828 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(327): Store=e301819a7ac81b4179f3ff5e1e4d8bad/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:12,828 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,831 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:12,831 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e301819a7ac81b4179f3ff5e1e4d8bad columnFamilyName B 2024-11-16T16:16:12,831 DEBUG [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:12,833 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(327): Store=e301819a7ac81b4179f3ff5e1e4d8bad/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:12,833 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,836 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:12,836 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e301819a7ac81b4179f3ff5e1e4d8bad columnFamilyName C 2024-11-16T16:16:12,836 DEBUG [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:12,837 INFO [StoreOpener-e301819a7ac81b4179f3ff5e1e4d8bad-1 {}] regionserver.HStore(327): Store=e301819a7ac81b4179f3ff5e1e4d8bad/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:12,837 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,837 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,838 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,838 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,839 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,839 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,840 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:16:12,841 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,843 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:16:12,843 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened e301819a7ac81b4179f3ff5e1e4d8bad; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59469449, jitterRate=-0.11383615434169769}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:16:12,843 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:12,844 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for e301819a7ac81b4179f3ff5e1e4d8bad: Running coprocessor pre-open hook at 1731773772819Writing region info on filesystem at 1731773772819Initializing all the Stores at 1731773772821 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773772821Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773772822 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773772822Cleaning up temporary data from old regions at 1731773772839 (+17 ms)Running coprocessor post-open hooks at 1731773772843 (+4 ms)Region opened successfully at 1731773772844 (+1 ms) 2024-11-16T16:16:12,845 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., pid=126, masterSystemTime=1731773772693 2024-11-16T16:16:12,847 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,847 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:12,848 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=e301819a7ac81b4179f3ff5e1e4d8bad, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:12,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:12,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-16T16:16:12,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 in 310 msec 2024-11-16T16:16:12,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-16T16:16:12,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, ASSIGN in 467 msec 2024-11-16T16:16:12,856 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:16:12,857 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773772856"}]},"ts":"1731773772856"} 2024-11-16T16:16:12,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:16:12,859 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:16:12,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3040 sec 2024-11-16T16:16:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-16T16:16:13,716 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:13,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,718 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,718 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55aa755a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,720 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,721 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48b774e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,722 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,725 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@614d0348 2024-11-16T16:16:13,726 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:16:13,727 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36618, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:16:13,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,729 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,729 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3270eee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,731 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17a11cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,733 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf1b0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,734 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d4bab34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,735 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,736 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,736 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,736 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5befbea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8139a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,738 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,739 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8b03ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ca27da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,741 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb21c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,742 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43fa3a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,743 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6727ff48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d617d54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,746 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74b2f96c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,746 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,747 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@772fc6c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,749 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,749 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,749 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58398256, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,751 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,751 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728bcc96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,752 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ab18f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,754 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28780aaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:13,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:13,755 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:13,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c2a8b85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:13,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:13,756 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:13,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6560cdf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:13,758 DEBUG [Thread-2103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,758 DEBUG [Thread-2112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,758 DEBUG [Thread-2116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,758 DEBUG [Thread-2101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,758 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,759 DEBUG [Thread-2120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-16T16:16:13,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,760 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,760 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,760 DEBUG [Thread-2107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,760 DEBUG [Thread-2109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:13,763 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:13,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,763 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37568, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,764 DEBUG [Thread-2118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,764 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:13,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:13,765 DEBUG [Thread-2105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,765 DEBUG [Thread-2114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,768 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:13,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:13,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,770 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,771 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:13,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2] 2024-11-16T16:16:13,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:13,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:13,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:13,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6bfc04c097414465b3fcd7e7a597b77c is 50, key is test_row_0/A:col10/1731773773781/Put/seqid=0 2024-11-16T16:16:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773833816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773833816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773833817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773833820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742296_1472 (size=12001) 2024-11-16T16:16:13,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773833827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:13,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:13,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:13,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:13,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:13,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:13,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773833935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773833935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773833935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773833935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,940 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:13,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:13,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773833944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:13,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773834145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773834145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773834145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773834145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773834155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6bfc04c097414465b3fcd7e7a597b77c 2024-11-16T16:16:14,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/e3e052a372b84391bb7dde53dd6fcf3c is 50, key is test_row_0/B:col10/1731773773781/Put/seqid=0 2024-11-16T16:16:14,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742297_1473 (size=12001) 2024-11-16T16:16:14,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:14,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773834455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773834456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773834456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773834456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773834465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/e3e052a372b84391bb7dde53dd6fcf3c 2024-11-16T16:16:14,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/8aaa600b821d4dc3ab2fad519e718097 is 50, key is test_row_0/C:col10/1731773773781/Put/seqid=0 2024-11-16T16:16:14,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742298_1474 (size=12001) 2024-11-16T16:16:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:14,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773834966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773834966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773834966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,973 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773834975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,982 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773834975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:14,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:14,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:14,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:15,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/8aaa600b821d4dc3ab2fad519e718097 2024-11-16T16:16:15,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6bfc04c097414465b3fcd7e7a597b77c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c 2024-11-16T16:16:15,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T16:16:15,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/e3e052a372b84391bb7dde53dd6fcf3c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c 2024-11-16T16:16:15,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T16:16:15,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/8aaa600b821d4dc3ab2fad519e718097 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097 2024-11-16T16:16:15,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097, entries=150, sequenceid=13, filesize=11.7 K 2024-11-16T16:16:15,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e301819a7ac81b4179f3ff5e1e4d8bad in 1315ms, sequenceid=13, compaction requested=false 2024-11-16T16:16:15,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:15,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:15,142 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/14ec667cb7bd4d638bb7063bdb5529f2 is 50, key is test_row_0/A:col10/1731773773802/Put/seqid=0 2024-11-16T16:16:15,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742299_1475 (size=12001) 2024-11-16T16:16:15,549 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/14ec667cb7bd4d638bb7063bdb5529f2 2024-11-16T16:16:15,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/51321b57c711419792cbd7710005c6e3 is 50, key is test_row_0/B:col10/1731773773802/Put/seqid=0 2024-11-16T16:16:15,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742300_1476 (size=12001) 2024-11-16T16:16:15,568 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/51321b57c711419792cbd7710005c6e3 2024-11-16T16:16:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/95312cd89911479f93d6f560541443a6 is 50, key is test_row_0/C:col10/1731773773802/Put/seqid=0 2024-11-16T16:16:15,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742301_1477 (size=12001) 2024-11-16T16:16:15,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:15,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:15,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:15,993 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/95312cd89911479f93d6f560541443a6 2024-11-16T16:16:15,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/14ec667cb7bd4d638bb7063bdb5529f2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2 2024-11-16T16:16:16,003 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:16:16,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/51321b57c711419792cbd7710005c6e3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3 2024-11-16T16:16:16,008 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773835995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773835996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773836000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,008 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773836002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773836004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,010 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/95312cd89911479f93d6f560541443a6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6 2024-11-16T16:16:16,019 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6, entries=150, sequenceid=33, filesize=11.7 K 2024-11-16T16:16:16,020 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e301819a7ac81b4179f3ff5e1e4d8bad in 878ms, sequenceid=33, compaction requested=false 2024-11-16T16:16:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-16T16:16:16,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-16T16:16:16,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-16T16:16:16,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-16T16:16:16,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.2660 sec 2024-11-16T16:16:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:16,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:16,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:16,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a04cdeda75b54676956d3fc999315675 is 50, key is test_row_0/A:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:16,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742302_1478 (size=14341) 2024-11-16T16:16:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773836146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773836156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773836157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,158 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773836157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773836265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773836265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773836266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773836274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773836475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773836485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773836485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773836495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a04cdeda75b54676956d3fc999315675 2024-11-16T16:16:16,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/1b46fababb184236a175374f4162a946 is 50, key is test_row_0/B:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:16,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742303_1479 (size=12001) 2024-11-16T16:16:16,688 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:16:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773836785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773836795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773836795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773836805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:16,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/1b46fababb184236a175374f4162a946 2024-11-16T16:16:16,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/89b296a65d704df1beb9624111f8d2b5 is 50, key is test_row_0/C:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:16,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742304_1480 (size=12001) 2024-11-16T16:16:17,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:17,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773837305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:17,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773837315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:17,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773837315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773837315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:17,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/89b296a65d704df1beb9624111f8d2b5 2024-11-16T16:16:17,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a04cdeda75b54676956d3fc999315675 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675 2024-11-16T16:16:17,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675, entries=200, sequenceid=46, filesize=14.0 K 2024-11-16T16:16:17,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/1b46fababb184236a175374f4162a946 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946 2024-11-16T16:16:17,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T16:16:17,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/89b296a65d704df1beb9624111f8d2b5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5 2024-11-16T16:16:17,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5, entries=150, sequenceid=46, filesize=11.7 K 2024-11-16T16:16:17,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for e301819a7ac81b4179f3ff5e1e4d8bad in 1252ms, sequenceid=46, compaction requested=true 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:17,371 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:17,371 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:17,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:17,372 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:17,372 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:17,372 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=37.4 K 2024-11-16T16:16:17,372 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=35.2 K 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e3e052a372b84391bb7dde53dd6fcf3c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773773770 2024-11-16T16:16:17,372 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bfc04c097414465b3fcd7e7a597b77c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773773770 2024-11-16T16:16:17,373 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 51321b57c711419792cbd7710005c6e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773773802 2024-11-16T16:16:17,373 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14ec667cb7bd4d638bb7063bdb5529f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773773802 2024-11-16T16:16:17,373 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b46fababb184236a175374f4162a946, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773776000 2024-11-16T16:16:17,373 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a04cdeda75b54676956d3fc999315675, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773775998 2024-11-16T16:16:17,379 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#407 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:17,379 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/7e2cf8129db043ecbdc82e83a0a71b89 is 50, key is test_row_0/A:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:17,383 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:17,383 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/9e2030f5b1684401ae0ea7a40e5d6365 is 50, key is test_row_0/B:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:17,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742305_1481 (size=12104) 2024-11-16T16:16:17,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742306_1482 (size=12104) 2024-11-16T16:16:17,816 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/9e2030f5b1684401ae0ea7a40e5d6365 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/9e2030f5b1684401ae0ea7a40e5d6365 2024-11-16T16:16:17,816 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/7e2cf8129db043ecbdc82e83a0a71b89 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/7e2cf8129db043ecbdc82e83a0a71b89 2024-11-16T16:16:17,819 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 7e2cf8129db043ecbdc82e83a0a71b89(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:17,819 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 9e2030f5b1684401ae0ea7a40e5d6365(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:17,819 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=13, startTime=1731773777371; duration=0sec 2024-11-16T16:16:17,819 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=13, startTime=1731773777371; duration=0sec 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:17,819 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:17,820 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:17,820 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:17,820 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:17,820 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=35.2 K 2024-11-16T16:16:17,821 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aaa600b821d4dc3ab2fad519e718097, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731773773770 2024-11-16T16:16:17,821 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95312cd89911479f93d6f560541443a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731773773802 2024-11-16T16:16:17,821 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89b296a65d704df1beb9624111f8d2b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773776000 2024-11-16T16:16:17,827 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:17,827 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/8b98e8eb26dc4905906be52b40e9ca6c is 50, key is test_row_0/C:col10/1731773776118/Put/seqid=0 2024-11-16T16:16:17,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742307_1483 (size=12104) 2024-11-16T16:16:17,833 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/8b98e8eb26dc4905906be52b40e9ca6c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8b98e8eb26dc4905906be52b40e9ca6c 2024-11-16T16:16:17,837 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into 8b98e8eb26dc4905906be52b40e9ca6c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:17,837 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:17,837 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=13, startTime=1731773777371; duration=0sec 2024-11-16T16:16:17,837 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:17,837 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-16T16:16:17,925 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:17,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-16T16:16:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:17,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:17,927 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:17,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:18,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:18,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/f9de891041894102b5cac7f901e0fdb1 is 50, key is test_row_0/A:col10/1731773776156/Put/seqid=0 2024-11-16T16:16:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:18,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742308_1484 (size=14341) 2024-11-16T16:16:18,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773838075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,081 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773838185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,190 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:18,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773838315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,322 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773838325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,334 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773838334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773838334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,340 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,340 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773838395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/f9de891041894102b5cac7f901e0fdb1 2024-11-16T16:16:18,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a58869ff4fd34ac0ba1b2a671ba69399 is 50, key is test_row_0/B:col10/1731773776156/Put/seqid=0 2024-11-16T16:16:18,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742309_1485 (size=12001) 2024-11-16T16:16:18,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:18,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773838705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:18,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:18,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:18,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a58869ff4fd34ac0ba1b2a671ba69399 2024-11-16T16:16:18,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b0e64a498d634edf8c2bfab8626fed25 is 50, key is test_row_0/C:col10/1731773776156/Put/seqid=0 2024-11-16T16:16:18,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742310_1486 (size=12001) 2024-11-16T16:16:18,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b0e64a498d634edf8c2bfab8626fed25 2024-11-16T16:16:18,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/f9de891041894102b5cac7f901e0fdb1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1 2024-11-16T16:16:18,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1, entries=200, sequenceid=69, filesize=14.0 K 2024-11-16T16:16:18,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a58869ff4fd34ac0ba1b2a671ba69399 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399 2024-11-16T16:16:18,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399, entries=150, sequenceid=69, filesize=11.7 K 2024-11-16T16:16:18,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b0e64a498d634edf8c2bfab8626fed25 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25 2024-11-16T16:16:18,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25, entries=150, sequenceid=69, filesize=11.7 K 2024-11-16T16:16:18,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e301819a7ac81b4179f3ff5e1e4d8bad in 838ms, sequenceid=69, compaction requested=false 2024-11-16T16:16:18,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:18,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-16T16:16:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:18,994 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:16:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:18,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a85054c4edfe45f99e9b38653368ad55 is 50, key is test_row_0/A:col10/1731773778068/Put/seqid=0 2024-11-16T16:16:19,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742311_1487 (size=12001) 2024-11-16T16:16:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:19,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:19,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:19,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773839334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:19,403 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a85054c4edfe45f99e9b38653368ad55 2024-11-16T16:16:19,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a32d8d41190d4644bef95d3073a448ea is 50, key is test_row_0/B:col10/1731773778068/Put/seqid=0 2024-11-16T16:16:19,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742312_1488 (size=12001) 2024-11-16T16:16:19,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:19,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773839445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,454 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:19,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773839664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:19,813 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a32d8d41190d4644bef95d3073a448ea 2024-11-16T16:16:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/bea1083bfa2c400baafc68ed517a1ef7 is 50, key is test_row_0/C:col10/1731773778068/Put/seqid=0 2024-11-16T16:16:19,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742313_1489 (size=12001) 2024-11-16T16:16:19,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773839975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:19,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:20,225 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/bea1083bfa2c400baafc68ed517a1ef7 2024-11-16T16:16:20,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a85054c4edfe45f99e9b38653368ad55 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55 2024-11-16T16:16:20,231 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55, entries=150, sequenceid=81, filesize=11.7 K 2024-11-16T16:16:20,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a32d8d41190d4644bef95d3073a448ea as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea 2024-11-16T16:16:20,235 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea, entries=150, sequenceid=81, filesize=11.7 K 2024-11-16T16:16:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/bea1083bfa2c400baafc68ed517a1ef7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7 2024-11-16T16:16:20,238 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7, entries=150, sequenceid=81, filesize=11.7 K 2024-11-16T16:16:20,239 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e301819a7ac81b4179f3ff5e1e4d8bad in 1245ms, sequenceid=81, compaction requested=true 2024-11-16T16:16:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:20,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-16T16:16:20,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-16T16:16:20,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-16T16:16:20,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3140 sec 2024-11-16T16:16:20,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3170 sec 2024-11-16T16:16:20,348 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T16:16:20,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:20,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:20,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:20,348 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:20,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:20,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:20,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:20,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/8c49b1e146024b6da59f810d8ab90204 is 50, key is test_row_0/A:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:20,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742314_1490 (size=14341) 2024-11-16T16:16:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773840364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773840367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,377 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773840374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773840375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773840485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773840485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773840495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773840495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773840705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773840705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,715 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773840715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:20,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/8c49b1e146024b6da59f810d8ab90204 2024-11-16T16:16:20,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/12d1a8627c764cfd8fce98dbd2189805 is 50, key is test_row_0/B:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:20,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742315_1491 (size=12001) 2024-11-16T16:16:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773841025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773841025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773841025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/12d1a8627c764cfd8fce98dbd2189805 2024-11-16T16:16:21,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/cee592dce8244c4bbe400fb6a226d788 is 50, key is test_row_0/C:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742316_1492 (size=12001) 2024-11-16T16:16:21,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773841515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,524 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773841545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773841545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773841545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:21,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/cee592dce8244c4bbe400fb6a226d788 2024-11-16T16:16:21,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/8c49b1e146024b6da59f810d8ab90204 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204 2024-11-16T16:16:21,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204, entries=200, sequenceid=102, filesize=14.0 K 2024-11-16T16:16:21,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/12d1a8627c764cfd8fce98dbd2189805 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805 2024-11-16T16:16:21,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T16:16:21,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/cee592dce8244c4bbe400fb6a226d788 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788 2024-11-16T16:16:21,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788, entries=150, sequenceid=102, filesize=11.7 K 2024-11-16T16:16:21,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e301819a7ac81b4179f3ff5e1e4d8bad in 1243ms, sequenceid=102, compaction requested=true 2024-11-16T16:16:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:21,592 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:21,592 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:21,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:21,593 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:21,593 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:21,593 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/9e2030f5b1684401ae0ea7a40e5d6365, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=47.0 K 2024-11-16T16:16:21,593 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/7e2cf8129db043ecbdc82e83a0a71b89, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=51.5 K 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2030f5b1684401ae0ea7a40e5d6365, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773776000 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e2cf8129db043ecbdc82e83a0a71b89, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773776000 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a58869ff4fd34ac0ba1b2a671ba69399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731773776155 2024-11-16T16:16:21,593 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9de891041894102b5cac7f901e0fdb1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731773776143 2024-11-16T16:16:21,594 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a32d8d41190d4644bef95d3073a448ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731773778064 2024-11-16T16:16:21,594 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a85054c4edfe45f99e9b38653368ad55, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731773778064 2024-11-16T16:16:21,594 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d1a8627c764cfd8fce98dbd2189805, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779326 2024-11-16T16:16:21,594 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c49b1e146024b6da59f810d8ab90204, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779320 2024-11-16T16:16:21,601 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:21,601 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/5eeb1f471aa54c0a8c3e9e761114ca0d is 50, key is test_row_0/A:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:21,601 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:21,602 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/b7eda098ea464cc7a757772433008b20 is 50, key is test_row_0/B:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:21,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742317_1493 (size=12241) 2024-11-16T16:16:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742318_1494 (size=12241) 2024-11-16T16:16:22,009 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/5eeb1f471aa54c0a8c3e9e761114ca0d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/5eeb1f471aa54c0a8c3e9e761114ca0d 2024-11-16T16:16:22,013 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 5eeb1f471aa54c0a8c3e9e761114ca0d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:22,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:22,013 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=12, startTime=1731773781591; duration=0sec 2024-11-16T16:16:22,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:22,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:22,013 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:22,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:22,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:22,014 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:22,014 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8b98e8eb26dc4905906be52b40e9ca6c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=47.0 K 2024-11-16T16:16:22,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b98e8eb26dc4905906be52b40e9ca6c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1731773776000 2024-11-16T16:16:22,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e64a498d634edf8c2bfab8626fed25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731773776155 2024-11-16T16:16:22,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting bea1083bfa2c400baafc68ed517a1ef7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731773778064 2024-11-16T16:16:22,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cee592dce8244c4bbe400fb6a226d788, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779326 2024-11-16T16:16:22,016 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/b7eda098ea464cc7a757772433008b20 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/b7eda098ea464cc7a757772433008b20 2024-11-16T16:16:22,019 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into b7eda098ea464cc7a757772433008b20(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:22,019 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:22,020 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=12, startTime=1731773781592; duration=0sec 2024-11-16T16:16:22,020 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:22,020 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:22,023 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#421 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:22,023 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/3fb61c6ee411470e91529a4f4d449f13 is 50, key is test_row_0/C:col10/1731773779333/Put/seqid=0 2024-11-16T16:16:22,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742319_1495 (size=12241) 2024-11-16T16:16:22,030 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/3fb61c6ee411470e91529a4f4d449f13 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/3fb61c6ee411470e91529a4f4d449f13 2024-11-16T16:16:22,033 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into 3fb61c6ee411470e91529a4f4d449f13(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:22,033 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:22,033 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=12, startTime=1731773781592; duration=0sec 2024-11-16T16:16:22,033 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:22,033 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-16T16:16:22,095 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:22,095 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-16T16:16:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:22,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:22,097 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:22,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:22,154 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-16T16:16:22,154 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-16T16:16:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:22,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-16T16:16:22,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:22,249 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:22,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:22,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a0940dba71c5490f98a4a8d7b242c1e1 is 50, key is test_row_0/A:col10/1731773780372/Put/seqid=0 2024-11-16T16:16:22,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742320_1496 (size=12001) 2024-11-16T16:16:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:22,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:22,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773782579, max timestamp=1731773782601Num uniques 859; 2024-11-16T16:16:22,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:22,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773782579, max timestamp=1731773782601Num uniques 859; 2024-11-16T16:16:22,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:22,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773782579, max timestamp=1731773782601Num uniques 859; 2024-11-16T16:16:22,604 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:22,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773842629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773842632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773842635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,657 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a0940dba71c5490f98a4a8d7b242c1e1 2024-11-16T16:16:22,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/2271316955b5495f92535d6c8deaf0a8 is 50, key is test_row_0/B:col10/1731773780372/Put/seqid=0 2024-11-16T16:16:22,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742321_1497 (size=12001) 2024-11-16T16:16:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:22,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773842745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773842745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773842745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773842955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773842955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:22,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773842955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:22,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,072 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/2271316955b5495f92535d6c8deaf0a8 2024-11-16T16:16:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/fd7ecdafee664ab2936633a8dc88dbe4 is 50, key is test_row_0/C:col10/1731773780372/Put/seqid=0 2024-11-16T16:16:23,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742322_1498 (size=12001) 2024-11-16T16:16:23,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773843265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773843265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773843275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,483 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/fd7ecdafee664ab2936633a8dc88dbe4 2024-11-16T16:16:23,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/a0940dba71c5490f98a4a8d7b242c1e1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1 2024-11-16T16:16:23,489 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T16:16:23,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/2271316955b5495f92535d6c8deaf0a8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8 2024-11-16T16:16:23,493 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T16:16:23,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/fd7ecdafee664ab2936633a8dc88dbe4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4 2024-11-16T16:16:23,499 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4, entries=150, sequenceid=116, filesize=11.7 K 2024-11-16T16:16:23,500 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e301819a7ac81b4179f3ff5e1e4d8bad in 1250ms, sequenceid=116, compaction requested=false 2024-11-16T16:16:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-16T16:16:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-16T16:16:23,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-16T16:16:23,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-16T16:16:23,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.4060 sec 2024-11-16T16:16:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:23,548 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:23,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:23,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf is 50, key is test_row_0/A:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:23,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742323_1499 (size=14541) 2024-11-16T16:16:23,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf 2024-11-16T16:16:23,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/0dd75d423c784bf68c14bbc368ce8f43 is 50, key is test_row_0/B:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:23,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742324_1500 (size=12151) 2024-11-16T16:16:23,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773843595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773843705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773843785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773843785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773843794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:23,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773843914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:23,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/0dd75d423c784bf68c14bbc368ce8f43 2024-11-16T16:16:23,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/5bead794764847d4933d9b312a416f58 is 50, key is test_row_0/C:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742325_1501 (size=12151) 2024-11-16T16:16:23,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/5bead794764847d4933d9b312a416f58 2024-11-16T16:16:23,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf 2024-11-16T16:16:23,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf, entries=200, sequenceid=151, filesize=14.2 K 2024-11-16T16:16:23,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/0dd75d423c784bf68c14bbc368ce8f43 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43 2024-11-16T16:16:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T16:16:24,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/5bead794764847d4933d9b312a416f58 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58 2024-11-16T16:16:24,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58, entries=150, sequenceid=151, filesize=11.9 K 2024-11-16T16:16:24,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e301819a7ac81b4179f3ff5e1e4d8bad in 456ms, sequenceid=151, compaction requested=true 2024-11-16T16:16:24,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:24,004 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:24,005 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:24,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:24,005 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:24,005 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:24,005 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,005 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/5eeb1f471aa54c0a8c3e9e761114ca0d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=37.9 K 2024-11-16T16:16:24,006 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5eeb1f471aa54c0a8c3e9e761114ca0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779326 2024-11-16T16:16:24,006 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0940dba71c5490f98a4a8d7b242c1e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731773780361 2024-11-16T16:16:24,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:24,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:24,006 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,006 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/b7eda098ea464cc7a757772433008b20, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=35.5 K 2024-11-16T16:16:24,006 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782615 2024-11-16T16:16:24,008 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b7eda098ea464cc7a757772433008b20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779326 2024-11-16T16:16:24,008 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 2271316955b5495f92535d6c8deaf0a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731773780361 2024-11-16T16:16:24,008 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd75d423c784bf68c14bbc368ce8f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782623 2024-11-16T16:16:24,013 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:24,014 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/889694a117c04ea78e8f964562170b81 is 50, key is test_row_0/A:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:24,016 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:24,016 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/785be80dbecc4f6e8adbc18dfd1106d1 is 50, key is test_row_0/B:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742326_1502 (size=12493) 2024-11-16T16:16:24,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742327_1503 (size=12493) 2024-11-16T16:16:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:24,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:24,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:24,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0de7ea490643441997c01913d603605d is 50, key is test_row_1/A:col10/1731773784235/Put/seqid=0 2024-11-16T16:16:24,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742328_1504 (size=12147) 2024-11-16T16:16:24,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-16T16:16:24,245 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:24,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0de7ea490643441997c01913d603605d 2024-11-16T16:16:24,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-16T16:16:24,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:24,251 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:24,251 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:24,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:24,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/566d911bcd8649e098fe97242942d249 is 50, key is test_row_1/B:col10/1731773784235/Put/seqid=0 2024-11-16T16:16:24,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742329_1505 (size=9757) 2024-11-16T16:16:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:24,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773784345, max timestamp=1731773784375Num uniques 859; 2024-11-16T16:16:24,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:24,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773784345, max timestamp=1731773784375Num uniques 859; 2024-11-16T16:16:24,387 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:24,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773784345, max timestamp=1731773784375Num uniques 859; 2024-11-16T16:16:24,388 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:24,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,429 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/889694a117c04ea78e8f964562170b81 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/889694a117c04ea78e8f964562170b81 2024-11-16T16:16:24,433 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/785be80dbecc4f6e8adbc18dfd1106d1 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/785be80dbecc4f6e8adbc18dfd1106d1 2024-11-16T16:16:24,443 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 889694a117c04ea78e8f964562170b81(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:24,443 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 785be80dbecc4f6e8adbc18dfd1106d1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:24,443 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=13, startTime=1731773784004; duration=0sec 2024-11-16T16:16:24,443 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=13, startTime=1731773784005; duration=0sec 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:24,443 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:24,444 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:24,444 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:24,444 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:24,444 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,444 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/3fb61c6ee411470e91529a4f4d449f13, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=35.5 K 2024-11-16T16:16:24,445 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb61c6ee411470e91529a4f4d449f13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731773779326 2024-11-16T16:16:24,445 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd7ecdafee664ab2936633a8dc88dbe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731773780361 2024-11-16T16:16:24,446 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bead794764847d4933d9b312a416f58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782623 2024-11-16T16:16:24,450 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:24,450 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/fda7f6f4063e43dd8a9a9f6037fa947d is 50, key is test_row_0/C:col10/1731773782623/Put/seqid=0 2024-11-16T16:16:24,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742330_1506 (size=12493) 2024-11-16T16:16:24,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773844476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773844477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T16:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:24,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:24,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773844585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,592 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773844595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/566d911bcd8649e098fe97242942d249 2024-11-16T16:16:24,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/4be25358696f445987f8d76a26c49665 is 50, key is test_row_1/C:col10/1731773784235/Put/seqid=0 2024-11-16T16:16:24,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742331_1507 (size=9757) 2024-11-16T16:16:24,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=162 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/4be25358696f445987f8d76a26c49665 2024-11-16T16:16:24,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0de7ea490643441997c01913d603605d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d 2024-11-16T16:16:24,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d, entries=150, sequenceid=162, filesize=11.9 K 2024-11-16T16:16:24,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/566d911bcd8649e098fe97242942d249 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249 2024-11-16T16:16:24,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249, entries=100, sequenceid=162, filesize=9.5 K 2024-11-16T16:16:24,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/4be25358696f445987f8d76a26c49665 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665 2024-11-16T16:16:24,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665, entries=100, sequenceid=162, filesize=9.5 K 2024-11-16T16:16:24,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e301819a7ac81b4179f3ff5e1e4d8bad in 441ms, sequenceid=162, compaction requested=false 2024-11-16T16:16:24,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:24,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-16T16:16:24,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:24,709 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:24,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:24,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/02f00081692141919c159ae292d5c54b is 50, key is test_row_0/A:col10/1731773784476/Put/seqid=0 2024-11-16T16:16:24,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742332_1508 (size=12151) 2024-11-16T16:16:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:24,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773844815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773844821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773844822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773844823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773844824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,857 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/fda7f6f4063e43dd8a9a9f6037fa947d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fda7f6f4063e43dd8a9a9f6037fa947d 2024-11-16T16:16:24,860 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into fda7f6f4063e43dd8a9a9f6037fa947d(size=12.2 K), total size for store is 21.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:24,861 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:24,861 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=13, startTime=1731773784005; duration=0sec 2024-11-16T16:16:24,861 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:24,861 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:24,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773844935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773844945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773844945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773844945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,948 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,116 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/02f00081692141919c159ae292d5c54b 2024-11-16T16:16:25,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/d0a928f00e05448aab84afb86d4b524c is 50, key is test_row_0/B:col10/1731773784476/Put/seqid=0 2024-11-16T16:16:25,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742333_1509 (size=12151) 2024-11-16T16:16:25,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773845135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773845158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773845158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773845158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773845158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,169 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773845475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773845475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773845475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773845475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,485 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,524 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/d0a928f00e05448aab84afb86d4b524c 2024-11-16T16:16:25,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/695a93194df04e5781e3edb7e284e6fc is 50, key is test_row_0/C:col10/1731773784476/Put/seqid=0 2024-11-16T16:16:25,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742334_1510 (size=12151) 2024-11-16T16:16:25,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773845645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:25,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:25,933 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/695a93194df04e5781e3edb7e284e6fc 2024-11-16T16:16:25,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/02f00081692141919c159ae292d5c54b as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b 2024-11-16T16:16:25,939 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b, entries=150, sequenceid=200, filesize=11.9 K 2024-11-16T16:16:25,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/d0a928f00e05448aab84afb86d4b524c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c 2024-11-16T16:16:25,942 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c, entries=150, sequenceid=200, filesize=11.9 K 2024-11-16T16:16:25,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/695a93194df04e5781e3edb7e284e6fc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc 2024-11-16T16:16:25,946 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc, entries=150, sequenceid=200, filesize=11.9 K 2024-11-16T16:16:25,947 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for e301819a7ac81b4179f3ff5e1e4d8bad in 1238ms, sequenceid=200, compaction requested=true 2024-11-16T16:16:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:25,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-16T16:16:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-16T16:16:25,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-16T16:16:25,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-16T16:16:25,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.7060 sec 2024-11-16T16:16:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:25,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:26,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 is 50, key is test_row_0/A:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:26,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742335_1511 (size=14541) 2024-11-16T16:16:26,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773846028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773846030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773846039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773846040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773846145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773846145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773846155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773846155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773846355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773846355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773846375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773846375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,384 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-16T16:16:26,395 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:26,395 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-16T16:16:26,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:26,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:26,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:26,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 2024-11-16T16:16:26,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/be5f031b63a342dc9bc3f317ad780d3a is 50, key is test_row_0/B:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:26,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742336_1512 (size=12151) 2024-11-16T16:16:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:26,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773846664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773846665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773846665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,673 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773846695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:26,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773846695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:26,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:26,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/be5f031b63a342dc9bc3f317ad780d3a 2024-11-16T16:16:26,825 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/85e95e517e06486e8ba5134b0ce7aeb8 is 50, key is test_row_0/C:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:26,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742337_1513 (size=12151) 2024-11-16T16:16:26,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:26,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:27,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:27,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773847185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:27,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773847185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773847215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:27,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773847215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:27,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:27,220 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:27,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/85e95e517e06486e8ba5134b0ce7aeb8 2024-11-16T16:16:27,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 2024-11-16T16:16:27,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9, entries=200, sequenceid=213, filesize=14.2 K 2024-11-16T16:16:27,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/be5f031b63a342dc9bc3f317ad780d3a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a 2024-11-16T16:16:27,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T16:16:27,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/85e95e517e06486e8ba5134b0ce7aeb8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8 2024-11-16T16:16:27,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8, entries=150, sequenceid=213, filesize=11.9 K 2024-11-16T16:16:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e301819a7ac81b4179f3ff5e1e4d8bad in 1249ms, sequenceid=213, compaction requested=true 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:27,246 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:27,246 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51332 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:27,247 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,247 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,247 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/785be80dbecc4f6e8adbc18dfd1106d1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=45.5 K 2024-11-16T16:16:27,247 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/889694a117c04ea78e8f964562170b81, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=50.1 K 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 889694a117c04ea78e8f964562170b81, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782623 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 785be80dbecc4f6e8adbc18dfd1106d1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782623 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0de7ea490643441997c01913d603605d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731773783590 2024-11-16T16:16:27,247 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566d911bcd8649e098fe97242942d249, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731773783593 2024-11-16T16:16:27,248 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f00081692141919c159ae292d5c54b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731773784461 2024-11-16T16:16:27,248 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a928f00e05448aab84afb86d4b524c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731773784461 2024-11-16T16:16:27,248 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 96c5f0b49c1844ab9d4a4fb3441a3ec9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784821 2024-11-16T16:16:27,248 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting be5f031b63a342dc9bc3f317ad780d3a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784822 2024-11-16T16:16:27,254 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:27,254 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:27,255 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/4b1d1f988d06421a8aa4a51e611d3401 is 50, key is test_row_0/B:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:27,255 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/10930be27dab41538d20212a4567cbf9 is 50, key is test_row_0/A:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:27,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742339_1515 (size=12629) 2024-11-16T16:16:27,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742338_1514 (size=12629) 2024-11-16T16:16:27,276 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/10930be27dab41538d20212a4567cbf9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/10930be27dab41538d20212a4567cbf9 2024-11-16T16:16:27,281 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 10930be27dab41538d20212a4567cbf9(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:27,281 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:27,281 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=12, startTime=1731773787246; duration=0sec 2024-11-16T16:16:27,281 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:27,281 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:27,281 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:27,282 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:27,282 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:27,282 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,282 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fda7f6f4063e43dd8a9a9f6037fa947d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=45.5 K 2024-11-16T16:16:27,282 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting fda7f6f4063e43dd8a9a9f6037fa947d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731773782623 2024-11-16T16:16:27,283 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4be25358696f445987f8d76a26c49665, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=162, earliestPutTs=1731773783593 2024-11-16T16:16:27,283 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 695a93194df04e5781e3edb7e284e6fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1731773784461 2024-11-16T16:16:27,283 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 85e95e517e06486e8ba5134b0ce7aeb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784822 2024-11-16T16:16:27,300 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:27,300 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/d85de1dcc0c34cd1be746531c603510a is 50, key is test_row_0/C:col10/1731773784822/Put/seqid=0 2024-11-16T16:16:27,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-16T16:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:27,312 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:27,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:27,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742340_1516 (size=12629) 2024-11-16T16:16:27,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/97c3148e969f418a8230b4424a5b88a9 is 50, key is test_row_0/A:col10/1731773786024/Put/seqid=0 2024-11-16T16:16:27,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742341_1517 (size=12151) 2024-11-16T16:16:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:27,665 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/4b1d1f988d06421a8aa4a51e611d3401 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4b1d1f988d06421a8aa4a51e611d3401 2024-11-16T16:16:27,669 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 4b1d1f988d06421a8aa4a51e611d3401(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:27,669 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:27,669 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=12, startTime=1731773787246; duration=0sec 2024-11-16T16:16:27,669 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:27,669 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:27,721 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/d85de1dcc0c34cd1be746531c603510a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/d85de1dcc0c34cd1be746531c603510a 2024-11-16T16:16:27,725 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into d85de1dcc0c34cd1be746531c603510a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:27,725 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:27,725 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=12, startTime=1731773787246; duration=0sec 2024-11-16T16:16:27,725 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:27,725 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:27,730 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/97c3148e969f418a8230b4424a5b88a9 2024-11-16T16:16:27,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/4039fb24315d4c638b204b600d1e0b1e is 50, key is test_row_0/B:col10/1731773786024/Put/seqid=0 2024-11-16T16:16:27,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742342_1518 (size=12151) 2024-11-16T16:16:28,141 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/4039fb24315d4c638b204b600d1e0b1e 2024-11-16T16:16:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/9ecb02e5e588463ebe234ff11c05322a is 50, key is test_row_0/C:col10/1731773786024/Put/seqid=0 2024-11-16T16:16:28,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742343_1519 (size=12151) 2024-11-16T16:16:28,150 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/9ecb02e5e588463ebe234ff11c05322a 2024-11-16T16:16:28,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/97c3148e969f418a8230b4424a5b88a9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9 2024-11-16T16:16:28,158 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T16:16:28,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/4039fb24315d4c638b204b600d1e0b1e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e 2024-11-16T16:16:28,163 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T16:16:28,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/9ecb02e5e588463ebe234ff11c05322a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a 2024-11-16T16:16:28,166 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a, entries=150, sequenceid=234, filesize=11.9 K 2024-11-16T16:16:28,167 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for e301819a7ac81b4179f3ff5e1e4d8bad in 855ms, sequenceid=234, compaction requested=false 2024-11-16T16:16:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:28,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-16T16:16:28,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-16T16:16:28,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-16T16:16:28,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-11-16T16:16:28,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.7750 sec 2024-11-16T16:16:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:28,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:28,212 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:28,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0fc03a05cecf4632807ba33d78d436ec is 50, key is test_row_0/A:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742344_1520 (size=19321) 2024-11-16T16:16:28,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773788261, max timestamp=1731773788278Num uniques 859; 2024-11-16T16:16:28,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731773788261, max timestamp=1731773788278Num uniques 859; 2024-11-16T16:16:28,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:28,281 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:28,282 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731773788261, max timestamp=1731773788278Num uniques 859; 2024-11-16T16:16:28,283 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:28,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773848315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773848328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773848329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773848330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,343 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773848435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773848435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773848436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773848454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-16T16:16:28,545 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:28,546 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-16T16:16:28,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:28,547 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:28,548 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:28,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:28,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0fc03a05cecf4632807ba33d78d436ec 2024-11-16T16:16:28,628 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/7e9fedddaf294817a90b044225274f58 is 50, key is test_row_0/B:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:28,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742345_1521 (size=12151) 2024-11-16T16:16:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773848645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773848645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773848645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:28,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773848665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,673 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:28,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773848695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,701 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:28,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:28,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773848965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773848965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773848965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773848985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:28,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:29,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:29,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/7e9fedddaf294817a90b044225274f58 2024-11-16T16:16:29,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f18ff00908d949229416790ac1af5b32 is 50, key is test_row_0/C:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:29,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742346_1522 (size=12151) 2024-11-16T16:16:29,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:29,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:29,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:29,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:29,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f18ff00908d949229416790ac1af5b32 2024-11-16T16:16:29,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/0fc03a05cecf4632807ba33d78d436ec as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec 2024-11-16T16:16:29,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec, entries=300, sequenceid=246, filesize=18.9 K 2024-11-16T16:16:29,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/7e9fedddaf294817a90b044225274f58 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58 2024-11-16T16:16:29,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58, entries=150, sequenceid=246, filesize=11.9 K 2024-11-16T16:16:29,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f18ff00908d949229416790ac1af5b32 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32 2024-11-16T16:16:29,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32, entries=150, sequenceid=246, filesize=11.9 K 2024-11-16T16:16:29,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for e301819a7ac81b4179f3ff5e1e4d8bad in 1243ms, sequenceid=246, compaction requested=true 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:29,455 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:29,455 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:29,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:29,456 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,456 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,456 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/10930be27dab41538d20212a4567cbf9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=43.1 K 2024-11-16T16:16:29,456 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4b1d1f988d06421a8aa4a51e611d3401, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.1 K 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1d1f988d06421a8aa4a51e611d3401, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784822 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10930be27dab41538d20212a4567cbf9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784822 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97c3148e969f418a8230b4424a5b88a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731773786024 2024-11-16T16:16:29,456 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4039fb24315d4c638b204b600d1e0b1e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731773786024 2024-11-16T16:16:29,457 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc03a05cecf4632807ba33d78d436ec, keycount=300, bloomtype=ROW, size=18.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:29,457 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9fedddaf294817a90b044225274f58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:29,462 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:29,462 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:29,462 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/82705904d9ef4e19b7e76e028aaa8b1a is 50, key is test_row_0/B:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:29,462 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/3d908e0d2c194d668771eb84ea58b4e7 is 50, key is test_row_0/A:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:29,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-16T16:16:29,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,463 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:29,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:29,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742347_1523 (size=12731) 2024-11-16T16:16:29,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742348_1524 (size=12731) 2024-11-16T16:16:29,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/227f4dd4e2c74493b91d3c7c0c6b02d9 is 50, key is test_row_0/A:col10/1731773788327/Put/seqid=0 2024-11-16T16:16:29,474 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/3d908e0d2c194d668771eb84ea58b4e7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/3d908e0d2c194d668771eb84ea58b4e7 2024-11-16T16:16:29,474 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/82705904d9ef4e19b7e76e028aaa8b1a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/82705904d9ef4e19b7e76e028aaa8b1a 2024-11-16T16:16:29,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742349_1525 (size=12301) 2024-11-16T16:16:29,478 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 3d908e0d2c194d668771eb84ea58b4e7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:29,478 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 82705904d9ef4e19b7e76e028aaa8b1a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:29,478 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=13, startTime=1731773789455; duration=0sec 2024-11-16T16:16:29,478 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=13, startTime=1731773789455; duration=0sec 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:29,478 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:29,479 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:29,479 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:29,479 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:29,479 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/d85de1dcc0c34cd1be746531c603510a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.1 K 2024-11-16T16:16:29,479 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d85de1dcc0c34cd1be746531c603510a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731773784822 2024-11-16T16:16:29,479 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ecb02e5e588463ebe234ff11c05322a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731773786024 2024-11-16T16:16:29,480 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f18ff00908d949229416790ac1af5b32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:29,484 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#452 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:29,484 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/05424124d5764802baa977fc16a943a9 is 50, key is test_row_0/C:col10/1731773788208/Put/seqid=0 2024-11-16T16:16:29,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742350_1526 (size=12731) 2024-11-16T16:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:29,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:29,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773849503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773849504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773849505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773849505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773849614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,621 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773849625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773849625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773849824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773849835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773849845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:29,875 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/227f4dd4e2c74493b91d3c7c0c6b02d9 2024-11-16T16:16:29,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/98a95e93a9bf419fb5a0ae1f2b7b1816 is 50, key is test_row_0/B:col10/1731773788327/Put/seqid=0 2024-11-16T16:16:29,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742351_1527 (size=12301) 2024-11-16T16:16:29,892 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/05424124d5764802baa977fc16a943a9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/05424124d5764802baa977fc16a943a9 2024-11-16T16:16:29,895 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into 05424124d5764802baa977fc16a943a9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:29,895 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:29,895 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=13, startTime=1731773789455; duration=0sec 2024-11-16T16:16:29,895 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:29,895 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:30,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773850145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773850154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,159 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773850164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,285 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/98a95e93a9bf419fb5a0ae1f2b7b1816 2024-11-16T16:16:30,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/7094a32912214842bb470da93045cb23 is 50, key is test_row_0/C:col10/1731773788327/Put/seqid=0 2024-11-16T16:16:30,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742352_1528 (size=12301) 2024-11-16T16:16:30,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773850525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,532 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773850655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773850664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:30,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773850685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:30,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:30,698 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/7094a32912214842bb470da93045cb23 2024-11-16T16:16:30,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/227f4dd4e2c74493b91d3c7c0c6b02d9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9 2024-11-16T16:16:30,704 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T16:16:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/98a95e93a9bf419fb5a0ae1f2b7b1816 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816 2024-11-16T16:16:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:30,707 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T16:16:30,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/7094a32912214842bb470da93045cb23 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23 2024-11-16T16:16:30,711 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23, entries=150, sequenceid=282, filesize=12.0 K 2024-11-16T16:16:30,712 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for e301819a7ac81b4179f3ff5e1e4d8bad in 1249ms, sequenceid=282, compaction requested=false 2024-11-16T16:16:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:30,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-16T16:16:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-16T16:16:30,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-16T16:16:30,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1650 sec 2024-11-16T16:16:30,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.1690 sec 2024-11-16T16:16:31,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:31,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:31,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:31,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/24d7c29ae4ed49e592bf2cdbd0e63d32 is 50, key is test_row_0/A:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:31,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742353_1529 (size=14741) 2024-11-16T16:16:31,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773851733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773791698, max timestamp=1731773791734Num uniques 859; 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773791698, max timestamp=1731773791734Num uniques 859; 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731773791698, max timestamp=1731773791734Num uniques 859; 2024-11-16T16:16:31,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-16T16:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:31,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773851783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:31,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773851784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,797 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:31,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773851845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773851895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773851904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773852065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/24d7c29ae4ed49e592bf2cdbd0e63d32 2024-11-16T16:16:32,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/6067370af0234972b1f5051c29d969cc is 50, key is test_row_0/B:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:32,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742354_1530 (size=12301) 2024-11-16T16:16:32,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773852115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773852124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773852375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773852435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773852435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/6067370af0234972b1f5051c29d969cc 2024-11-16T16:16:32,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f701de24b8f740ef87f69ec975332cdb is 50, key is test_row_0/C:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:32,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742355_1531 (size=12301) 2024-11-16T16:16:32,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f701de24b8f740ef87f69ec975332cdb 2024-11-16T16:16:32,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/24d7c29ae4ed49e592bf2cdbd0e63d32 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32 2024-11-16T16:16:32,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32, entries=200, sequenceid=296, filesize=14.4 K 2024-11-16T16:16:32,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/6067370af0234972b1f5051c29d969cc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc 2024-11-16T16:16:32,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T16:16:32,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f701de24b8f740ef87f69ec975332cdb as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb 2024-11-16T16:16:32,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb, entries=150, sequenceid=296, filesize=12.0 K 2024-11-16T16:16:32,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for e301819a7ac81b4179f3ff5e1e4d8bad in 839ms, sequenceid=296, compaction requested=true 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:32,519 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:32,519 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:32,519 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:32,520 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:32,520 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:32,521 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:32,521 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:32,521 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/82705904d9ef4e19b7e76e028aaa8b1a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.5 K 2024-11-16T16:16:32,521 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/3d908e0d2c194d668771eb84ea58b4e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=38.8 K 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 82705904d9ef4e19b7e76e028aaa8b1a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d908e0d2c194d668771eb84ea58b4e7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 98a95e93a9bf419fb5a0ae1f2b7b1816, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731773788327 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227f4dd4e2c74493b91d3c7c0c6b02d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731773788327 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6067370af0234972b1f5051c29d969cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:32,521 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d7c29ae4ed49e592bf2cdbd0e63d32, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:32,527 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:32,527 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/5be1504cbc8e4a76ad6949b25896a09f is 50, key is test_row_0/B:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:32,528 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:32,528 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/cef64261f4a34ca58ff41a81fd8a82a7 is 50, key is test_row_0/A:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:32,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742356_1532 (size=12983) 2024-11-16T16:16:32,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742357_1533 (size=12983) 2024-11-16T16:16:32,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:32,558 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-16T16:16:32,558 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:32,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:32,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/b4c71498394744e0bfbd22672c2380ca is 50, key is test_row_0/A:col10/1731773792557/Put/seqid=0 2024-11-16T16:16:32,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742358_1534 (size=14741) 2024-11-16T16:16:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773852608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-16T16:16:32,715 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:32,715 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:32,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-16T16:16:32,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:32,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:32,717 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:32,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37576 deadline: 1731773852725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773852725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,734 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in e301819a7ac81b4179f3ff5e1e4d8bad of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,735 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:32,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:32,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:32,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:32,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37522 deadline: 1731773852895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,939 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/5be1504cbc8e4a76ad6949b25896a09f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/5be1504cbc8e4a76ad6949b25896a09f 2024-11-16T16:16:32,940 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/cef64261f4a34ca58ff41a81fd8a82a7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/cef64261f4a34ca58ff41a81fd8a82a7 2024-11-16T16:16:32,944 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 5be1504cbc8e4a76ad6949b25896a09f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:32,944 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into cef64261f4a34ca58ff41a81fd8a82a7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:32,944 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=13, startTime=1731773792519; duration=0sec 2024-11-16T16:16:32,944 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=13, startTime=1731773792519; duration=0sec 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:32,944 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:32,947 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:32,947 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:32,947 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:32,948 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/05424124d5764802baa977fc16a943a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.5 K 2024-11-16T16:16:32,948 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 05424124d5764802baa977fc16a943a9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731773788206 2024-11-16T16:16:32,948 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7094a32912214842bb470da93045cb23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731773788327 2024-11-16T16:16:32,948 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f701de24b8f740ef87f69ec975332cdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:32,954 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:32,954 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/6a04672f14d54a8c9b450fb2c1d2fd05 is 50, key is test_row_0/C:col10/1731773789503/Put/seqid=0 2024-11-16T16:16:32,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773852945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742359_1535 (size=12983) 2024-11-16T16:16:32,961 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/6a04672f14d54a8c9b450fb2c1d2fd05 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/6a04672f14d54a8c9b450fb2c1d2fd05 2024-11-16T16:16:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37520 deadline: 1731773852955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:32,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37592 deadline: 1731773852955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:32,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:32,966 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into 6a04672f14d54a8c9b450fb2c1d2fd05(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:32,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/b4c71498394744e0bfbd22672c2380ca 2024-11-16T16:16:32,966 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:32,966 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=13, startTime=1731773792519; duration=0sec 2024-11-16T16:16:32,966 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:32,966 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:32,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/3e2de2bfd8cf46b8afdb95b3d01e2649 is 50, key is test_row_0/B:col10/1731773792557/Put/seqid=0 2024-11-16T16:16:32,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742360_1536 (size=12301) 2024-11-16T16:16:33,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:33,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,273 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:33,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773853265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:33,273 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:33,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:33,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/3e2de2bfd8cf46b8afdb95b3d01e2649 2024-11-16T16:16:33,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/c847e5e221ab438a98a0fdb407feba30 is 50, key is test_row_0/C:col10/1731773792557/Put/seqid=0 2024-11-16T16:16:33,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742361_1537 (size=12301) 2024-11-16T16:16:33,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,769 INFO [Thread-2116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2116. 2024-11-16T16:16:33,769 DEBUG [Thread-2116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,769 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,769 DEBUG [Thread-2116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,769 INFO [Thread-2114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2114. 2024-11-16T16:16:33,769 DEBUG [Thread-2114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,769 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,770 DEBUG [Thread-2114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,770 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,771 INFO [Thread-2120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2120. 2024-11-16T16:16:33,772 DEBUG [Thread-2120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,772 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,772 DEBUG [Thread-2120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,772 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,779 INFO [Thread-2112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2112. 2024-11-16T16:16:33,779 DEBUG [Thread-2112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,779 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,779 DEBUG [Thread-2112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,780 INFO [Thread-2118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2118. 2024-11-16T16:16:33,780 DEBUG [Thread-2118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,780 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,780 DEBUG [Thread-2118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/c847e5e221ab438a98a0fdb407feba30 2024-11-16T16:16:33,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:37614 deadline: 1731773853785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 , the old value is region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:33,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:33,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., hostname=303fe01965e8,37355,1731773652566, seqNum=2 because the exception is null or not the one we care about 2024-11-16T16:16:33,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/b4c71498394744e0bfbd22672c2380ca as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca 2024-11-16T16:16:33,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca, entries=200, sequenceid=331, filesize=14.4 K 2024-11-16T16:16:33,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/3e2de2bfd8cf46b8afdb95b3d01e2649 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649 2024-11-16T16:16:33,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T16:16:33,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/c847e5e221ab438a98a0fdb407feba30 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30 2024-11-16T16:16:33,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30, entries=150, sequenceid=331, filesize=12.0 K 2024-11-16T16:16:33,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for e301819a7ac81b4179f3ff5e1e4d8bad in 1242ms, sequenceid=331, compaction requested=false 2024-11-16T16:16:33,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:33,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:33,918 INFO [Thread-2103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2103. 2024-11-16T16:16:33,918 DEBUG [Thread-2103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,919 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:33,919 DEBUG [Thread-2103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:33,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:33,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:33,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/d1930216660e438ba2d5aa1e2dfd6ec8 is 50, key is test_row_0/A:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:33,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742362_1538 (size=12301) 2024-11-16T16:16:33,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:33,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:33,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:33,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:33,978 INFO [Thread-2101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2101. 2024-11-16T16:16:33,978 INFO [Thread-2107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2107. 2024-11-16T16:16:33,979 DEBUG [Thread-2101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,979 DEBUG [Thread-2107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:33,979 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,979 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,979 DEBUG [Thread-2101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,979 DEBUG [Thread-2107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:33,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:33,979 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:34,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/d1930216660e438ba2d5aa1e2dfd6ec8 2024-11-16T16:16:34,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a7e8a5faf09a40ec93a3709e00dce3d8 is 50, key is test_row_0/B:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:34,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742363_1539 (size=12301) 2024-11-16T16:16:34,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a7e8a5faf09a40ec93a3709e00dce3d8 2024-11-16T16:16:34,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b7038926ce494dacb8f9620a438c2910 is 50, key is test_row_0/C:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:34,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742364_1540 (size=12301) 2024-11-16T16:16:34,798 INFO [Thread-2105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2105. 2024-11-16T16:16:34,799 DEBUG [Thread-2105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:34,799 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:34,799 DEBUG [Thread-2105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:34,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:34,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:34,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:34,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b7038926ce494dacb8f9620a438c2910 2024-11-16T16:16:35,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/d1930216660e438ba2d5aa1e2dfd6ec8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8 2024-11-16T16:16:35,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T16:16:35,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/a7e8a5faf09a40ec93a3709e00dce3d8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8 2024-11-16T16:16:35,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:35,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. as already flushing 2024-11-16T16:16:35,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T16:16:35,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:35,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/b7038926ce494dacb8f9620a438c2910 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910 2024-11-16T16:16:35,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910, entries=150, sequenceid=345, filesize=12.0 K 2024-11-16T16:16:35,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for e301819a7ac81b4179f3ff5e1e4d8bad in 1266ms, sequenceid=345, compaction requested=true 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:35,185 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:35,185 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e301819a7ac81b4179f3ff5e1e4d8bad:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:35,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/A is initiating minor compaction (all files) 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/B is initiating minor compaction (all files) 2024-11-16T16:16:35,186 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/A in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,186 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/B in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,186 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/cef64261f4a34ca58ff41a81fd8a82a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=39.1 K 2024-11-16T16:16:35,186 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/5be1504cbc8e4a76ad6949b25896a09f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.7 K 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef64261f4a34ca58ff41a81fd8a82a7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 5be1504cbc8e4a76ad6949b25896a09f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:35,186 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c71498394744e0bfbd22672c2380ca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731773791769 2024-11-16T16:16:35,187 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e2de2bfd8cf46b8afdb95b3d01e2649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731773791769 2024-11-16T16:16:35,187 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e8a5faf09a40ec93a3709e00dce3d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773792593 2024-11-16T16:16:35,187 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1930216660e438ba2d5aa1e2dfd6ec8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773792593 2024-11-16T16:16:35,194 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#A#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:35,194 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/4278954b53854184a2b1bb784d7591f4 is 50, key is test_row_0/A:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:35,195 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#B#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:35,195 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/21532528d65e40398332230ce2f2a47e is 50, key is test_row_0/B:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:35,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742365_1541 (size=13085) 2024-11-16T16:16:35,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742366_1542 (size=13085) 2024-11-16T16:16:35,205 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/21532528d65e40398332230ce2f2a47e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/21532528d65e40398332230ce2f2a47e 2024-11-16T16:16:35,208 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/B of e301819a7ac81b4179f3ff5e1e4d8bad into 21532528d65e40398332230ce2f2a47e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:35,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:35,208 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/B, priority=13, startTime=1731773795185; duration=0sec 2024-11-16T16:16:35,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:35,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:B 2024-11-16T16:16:35,208 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:35,209 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:35,209 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): e301819a7ac81b4179f3ff5e1e4d8bad/C is initiating minor compaction (all files) 2024-11-16T16:16:35,209 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e301819a7ac81b4179f3ff5e1e4d8bad/C in TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,209 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/6a04672f14d54a8c9b450fb2c1d2fd05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp, totalSize=36.7 K 2024-11-16T16:16:35,209 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a04672f14d54a8c9b450fb2c1d2fd05, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731773789496 2024-11-16T16:16:35,210 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c847e5e221ab438a98a0fdb407feba30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731773791769 2024-11-16T16:16:35,210 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b7038926ce494dacb8f9620a438c2910, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1731773792593 2024-11-16T16:16:35,215 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e301819a7ac81b4179f3ff5e1e4d8bad#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:35,215 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f29ff4a1314142fdafe7cb62627be966 is 50, key is test_row_0/C:col10/1731773792607/Put/seqid=0 2024-11-16T16:16:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742367_1543 (size=13085) 2024-11-16T16:16:35,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-16T16:16:35,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:35,333 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:35,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:35,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6fa21830d9a84acd81bd4f9a9636811a is 50, key is test_row_0/A:col10/1731773793976/Put/seqid=0 2024-11-16T16:16:35,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742368_1544 (size=9857) 2024-11-16T16:16:35,605 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/4278954b53854184a2b1bb784d7591f4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/4278954b53854184a2b1bb784d7591f4 2024-11-16T16:16:35,611 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/A of e301819a7ac81b4179f3ff5e1e4d8bad into 4278954b53854184a2b1bb784d7591f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:35,611 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:35,611 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/A, priority=13, startTime=1731773795185; duration=0sec 2024-11-16T16:16:35,611 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:35,611 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:A 2024-11-16T16:16:35,621 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/f29ff4a1314142fdafe7cb62627be966 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f29ff4a1314142fdafe7cb62627be966 2024-11-16T16:16:35,624 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e301819a7ac81b4179f3ff5e1e4d8bad/C of e301819a7ac81b4179f3ff5e1e4d8bad into f29ff4a1314142fdafe7cb62627be966(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:35,624 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:35,624 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., storeName=e301819a7ac81b4179f3ff5e1e4d8bad/C, priority=13, startTime=1731773795185; duration=0sec 2024-11-16T16:16:35,624 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:35,624 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e301819a7ac81b4179f3ff5e1e4d8bad:C 2024-11-16T16:16:35,745 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6fa21830d9a84acd81bd4f9a9636811a 2024-11-16T16:16:35,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/61f726219edb4581972e89bfcc7b74e9 is 50, key is test_row_0/B:col10/1731773793976/Put/seqid=0 2024-11-16T16:16:35,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742369_1545 (size=9857) 2024-11-16T16:16:36,161 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/61f726219edb4581972e89bfcc7b74e9 2024-11-16T16:16:36,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/a0d4f0c5cf7e4df591058d1f041a6a6d is 50, key is test_row_0/C:col10/1731773793976/Put/seqid=0 2024-11-16T16:16:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742370_1546 (size=9857) 2024-11-16T16:16:36,582 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/a0d4f0c5cf7e4df591058d1f041a6a6d 2024-11-16T16:16:36,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/6fa21830d9a84acd81bd4f9a9636811a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6fa21830d9a84acd81bd4f9a9636811a 2024-11-16T16:16:36,596 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6fa21830d9a84acd81bd4f9a9636811a, entries=100, sequenceid=352, filesize=9.6 K 2024-11-16T16:16:36,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/61f726219edb4581972e89bfcc7b74e9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/61f726219edb4581972e89bfcc7b74e9 2024-11-16T16:16:36,600 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/61f726219edb4581972e89bfcc7b74e9, entries=100, sequenceid=352, filesize=9.6 K 2024-11-16T16:16:36,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/a0d4f0c5cf7e4df591058d1f041a6a6d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/a0d4f0c5cf7e4df591058d1f041a6a6d 2024-11-16T16:16:36,605 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/a0d4f0c5cf7e4df591058d1f041a6a6d, entries=100, sequenceid=352, filesize=9.6 K 2024-11-16T16:16:36,606 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for e301819a7ac81b4179f3ff5e1e4d8bad in 1273ms, sequenceid=352, compaction requested=false 2024-11-16T16:16:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for e301819a7ac81b4179f3ff5e1e4d8bad: 2024-11-16T16:16:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:36,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-16T16:16:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-16T16:16:36,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-16T16:16:36,609 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8900 sec 2024-11-16T16:16:36,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 3.8950 sec 2024-11-16T16:16:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-16T16:16:36,875 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:40,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:16:42,789 INFO [Thread-2109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2109. 2024-11-16T16:16:42,789 DEBUG [Thread-2109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:16:42,789 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:42,789 DEBUG [Thread-2109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:42,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1261 2024-11-16T16:16:42,790 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3777 rows 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1258 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3773 rows 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1256 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3765 rows 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1251 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3752 rows 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1247 2024-11-16T16:16:42,791 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3738 rows 2024-11-16T16:16:42,791 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:16:42,791 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:16:42,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:42,792 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:42,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:16:42,794 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:16:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:42,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:42,799 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773802798"}]},"ts":"1731773802798"} 2024-11-16T16:16:42,800 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:16:42,800 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:16:42,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:16:42,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, UNASSIGN}] 2024-11-16T16:16:42,804 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, UNASSIGN 2024-11-16T16:16:42,805 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e301819a7ac81b4179f3ff5e1e4d8bad, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:42,806 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, UNASSIGN because future has completed 2024-11-16T16:16:42,806 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:16:42,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:42,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:42,960 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:42,960 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:16:42,960 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing e301819a7ac81b4179f3ff5e1e4d8bad, disabling compactions & flushes 2024-11-16T16:16:42,960 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:42,960 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:42,960 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. after waiting 0 ms 2024-11-16T16:16:42,961 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:42,961 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing e301819a7ac81b4179f3ff5e1e4d8bad 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-16T16:16:42,961 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=A 2024-11-16T16:16:42,961 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:42,962 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=B 2024-11-16T16:16:42,962 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:42,962 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e301819a7ac81b4179f3ff5e1e4d8bad, store=C 2024-11-16T16:16:42,962 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:42,971 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/1f823683568b4dbf90ad74f2e01a35d8 is 50, key is test_row_0/A:col10/1731773802786/Put/seqid=0 2024-11-16T16:16:42,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742371_1547 (size=7415) 2024-11-16T16:16:43,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:43,376 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/1f823683568b4dbf90ad74f2e01a35d8 2024-11-16T16:16:43,390 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/40a55d921e244186b96e37e88c7b1586 is 50, key is test_row_0/B:col10/1731773802786/Put/seqid=0 2024-11-16T16:16:43,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742372_1548 (size=7415) 2024-11-16T16:16:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:43,794 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/40a55d921e244186b96e37e88c7b1586 2024-11-16T16:16:43,806 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/e7baeaa7679641729845bec7c496db6d is 50, key is test_row_0/C:col10/1731773802786/Put/seqid=0 2024-11-16T16:16:43,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742373_1549 (size=7415) 2024-11-16T16:16:43,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:44,210 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/e7baeaa7679641729845bec7c496db6d 2024-11-16T16:16:44,213 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/A/1f823683568b4dbf90ad74f2e01a35d8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/1f823683568b4dbf90ad74f2e01a35d8 2024-11-16T16:16:44,216 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/1f823683568b4dbf90ad74f2e01a35d8, entries=50, sequenceid=358, filesize=7.2 K 2024-11-16T16:16:44,217 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/B/40a55d921e244186b96e37e88c7b1586 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/40a55d921e244186b96e37e88c7b1586 2024-11-16T16:16:44,219 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/40a55d921e244186b96e37e88c7b1586, entries=50, sequenceid=358, filesize=7.2 K 2024-11-16T16:16:44,220 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/.tmp/C/e7baeaa7679641729845bec7c496db6d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/e7baeaa7679641729845bec7c496db6d 2024-11-16T16:16:44,222 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/e7baeaa7679641729845bec7c496db6d, entries=50, sequenceid=358, filesize=7.2 K 2024-11-16T16:16:44,223 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for e301819a7ac81b4179f3ff5e1e4d8bad in 1262ms, sequenceid=358, compaction requested=true 2024-11-16T16:16:44,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/7e2cf8129db043ecbdc82e83a0a71b89, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/5eeb1f471aa54c0a8c3e9e761114ca0d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/889694a117c04ea78e8f964562170b81, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/10930be27dab41538d20212a4567cbf9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/3d908e0d2c194d668771eb84ea58b4e7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/cef64261f4a34ca58ff41a81fd8a82a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8] to archive 2024-11-16T16:16:44,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:44,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6bfc04c097414465b3fcd7e7a597b77c 2024-11-16T16:16:44,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/14ec667cb7bd4d638bb7063bdb5529f2 2024-11-16T16:16:44,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a04cdeda75b54676956d3fc999315675 2024-11-16T16:16:44,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/7e2cf8129db043ecbdc82e83a0a71b89 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/7e2cf8129db043ecbdc82e83a0a71b89 2024-11-16T16:16:44,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/f9de891041894102b5cac7f901e0fdb1 2024-11-16T16:16:44,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a85054c4edfe45f99e9b38653368ad55 2024-11-16T16:16:44,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/8c49b1e146024b6da59f810d8ab90204 2024-11-16T16:16:44,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/5eeb1f471aa54c0a8c3e9e761114ca0d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/5eeb1f471aa54c0a8c3e9e761114ca0d 2024-11-16T16:16:44,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/a0940dba71c5490f98a4a8d7b242c1e1 2024-11-16T16:16:44,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/2b2bcdfd5d0a4c0fa12ebe1c7c3a5fcf 2024-11-16T16:16:44,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/889694a117c04ea78e8f964562170b81 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/889694a117c04ea78e8f964562170b81 2024-11-16T16:16:44,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0de7ea490643441997c01913d603605d 2024-11-16T16:16:44,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/02f00081692141919c159ae292d5c54b 2024-11-16T16:16:44,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/96c5f0b49c1844ab9d4a4fb3441a3ec9 2024-11-16T16:16:44,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/10930be27dab41538d20212a4567cbf9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/10930be27dab41538d20212a4567cbf9 2024-11-16T16:16:44,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/97c3148e969f418a8230b4424a5b88a9 2024-11-16T16:16:44,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/0fc03a05cecf4632807ba33d78d436ec 2024-11-16T16:16:44,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/3d908e0d2c194d668771eb84ea58b4e7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/3d908e0d2c194d668771eb84ea58b4e7 2024-11-16T16:16:44,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/227f4dd4e2c74493b91d3c7c0c6b02d9 2024-11-16T16:16:44,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/24d7c29ae4ed49e592bf2cdbd0e63d32 2024-11-16T16:16:44,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/cef64261f4a34ca58ff41a81fd8a82a7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/cef64261f4a34ca58ff41a81fd8a82a7 2024-11-16T16:16:44,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/b4c71498394744e0bfbd22672c2380ca 2024-11-16T16:16:44,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/d1930216660e438ba2d5aa1e2dfd6ec8 2024-11-16T16:16:44,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/9e2030f5b1684401ae0ea7a40e5d6365, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/b7eda098ea464cc7a757772433008b20, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/785be80dbecc4f6e8adbc18dfd1106d1, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4b1d1f988d06421a8aa4a51e611d3401, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/82705904d9ef4e19b7e76e028aaa8b1a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/5be1504cbc8e4a76ad6949b25896a09f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8] to archive 2024-11-16T16:16:44,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:44,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/e3e052a372b84391bb7dde53dd6fcf3c 2024-11-16T16:16:44,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/51321b57c711419792cbd7710005c6e3 2024-11-16T16:16:44,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/9e2030f5b1684401ae0ea7a40e5d6365 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/9e2030f5b1684401ae0ea7a40e5d6365 2024-11-16T16:16:44,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/1b46fababb184236a175374f4162a946 2024-11-16T16:16:44,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a58869ff4fd34ac0ba1b2a671ba69399 2024-11-16T16:16:44,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a32d8d41190d4644bef95d3073a448ea 2024-11-16T16:16:44,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/b7eda098ea464cc7a757772433008b20 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/b7eda098ea464cc7a757772433008b20 2024-11-16T16:16:44,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/12d1a8627c764cfd8fce98dbd2189805 2024-11-16T16:16:44,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/2271316955b5495f92535d6c8deaf0a8 2024-11-16T16:16:44,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/785be80dbecc4f6e8adbc18dfd1106d1 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/785be80dbecc4f6e8adbc18dfd1106d1 2024-11-16T16:16:44,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/0dd75d423c784bf68c14bbc368ce8f43 2024-11-16T16:16:44,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/566d911bcd8649e098fe97242942d249 2024-11-16T16:16:44,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/d0a928f00e05448aab84afb86d4b524c 2024-11-16T16:16:44,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4b1d1f988d06421a8aa4a51e611d3401 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4b1d1f988d06421a8aa4a51e611d3401 2024-11-16T16:16:44,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/be5f031b63a342dc9bc3f317ad780d3a 2024-11-16T16:16:44,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/4039fb24315d4c638b204b600d1e0b1e 2024-11-16T16:16:44,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/82705904d9ef4e19b7e76e028aaa8b1a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/82705904d9ef4e19b7e76e028aaa8b1a 2024-11-16T16:16:44,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/7e9fedddaf294817a90b044225274f58 2024-11-16T16:16:44,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/98a95e93a9bf419fb5a0ae1f2b7b1816 2024-11-16T16:16:44,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/5be1504cbc8e4a76ad6949b25896a09f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/5be1504cbc8e4a76ad6949b25896a09f 2024-11-16T16:16:44,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/6067370af0234972b1f5051c29d969cc 2024-11-16T16:16:44,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/3e2de2bfd8cf46b8afdb95b3d01e2649 2024-11-16T16:16:44,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/a7e8a5faf09a40ec93a3709e00dce3d8 2024-11-16T16:16:44,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8b98e8eb26dc4905906be52b40e9ca6c, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/3fb61c6ee411470e91529a4f4d449f13, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fda7f6f4063e43dd8a9a9f6037fa947d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/d85de1dcc0c34cd1be746531c603510a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/05424124d5764802baa977fc16a943a9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/6a04672f14d54a8c9b450fb2c1d2fd05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910] to archive 2024-11-16T16:16:44,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:16:44,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8aaa600b821d4dc3ab2fad519e718097 2024-11-16T16:16:44,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/95312cd89911479f93d6f560541443a6 2024-11-16T16:16:44,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8b98e8eb26dc4905906be52b40e9ca6c to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/8b98e8eb26dc4905906be52b40e9ca6c 2024-11-16T16:16:44,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/89b296a65d704df1beb9624111f8d2b5 2024-11-16T16:16:44,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b0e64a498d634edf8c2bfab8626fed25 2024-11-16T16:16:44,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/bea1083bfa2c400baafc68ed517a1ef7 2024-11-16T16:16:44,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/3fb61c6ee411470e91529a4f4d449f13 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/3fb61c6ee411470e91529a4f4d449f13 2024-11-16T16:16:44,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/cee592dce8244c4bbe400fb6a226d788 2024-11-16T16:16:44,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fd7ecdafee664ab2936633a8dc88dbe4 2024-11-16T16:16:44,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fda7f6f4063e43dd8a9a9f6037fa947d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/fda7f6f4063e43dd8a9a9f6037fa947d 2024-11-16T16:16:44,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/5bead794764847d4933d9b312a416f58 2024-11-16T16:16:44,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/4be25358696f445987f8d76a26c49665 2024-11-16T16:16:44,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/695a93194df04e5781e3edb7e284e6fc 2024-11-16T16:16:44,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/d85de1dcc0c34cd1be746531c603510a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/d85de1dcc0c34cd1be746531c603510a 2024-11-16T16:16:44,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/85e95e517e06486e8ba5134b0ce7aeb8 2024-11-16T16:16:44,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/9ecb02e5e588463ebe234ff11c05322a 2024-11-16T16:16:44,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/05424124d5764802baa977fc16a943a9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/05424124d5764802baa977fc16a943a9 2024-11-16T16:16:44,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f18ff00908d949229416790ac1af5b32 2024-11-16T16:16:44,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/7094a32912214842bb470da93045cb23 2024-11-16T16:16:44,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/6a04672f14d54a8c9b450fb2c1d2fd05 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/6a04672f14d54a8c9b450fb2c1d2fd05 2024-11-16T16:16:44,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f701de24b8f740ef87f69ec975332cdb 2024-11-16T16:16:44,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/c847e5e221ab438a98a0fdb407feba30 2024-11-16T16:16:44,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/b7038926ce494dacb8f9620a438c2910 2024-11-16T16:16:44,291 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/recovered.edits/361.seqid, newMaxSeqId=361, maxSeqId=1 2024-11-16T16:16:44,291 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. 2024-11-16T16:16:44,292 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for e301819a7ac81b4179f3ff5e1e4d8bad: Waiting for close lock at 1731773802960Running coprocessor pre-close hooks at 1731773802960Disabling compacts and flushes for region at 1731773802960Disabling writes for close at 1731773802960Obtaining lock to block concurrent updates at 1731773802961 (+1 ms)Preparing flush snapshotting stores in e301819a7ac81b4179f3ff5e1e4d8bad at 1731773802961Finished memstore snapshotting TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=21120, getOffHeapSize=0, getCellsCount=150 at 1731773802962 (+1 ms)Flushing stores of TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad. at 1731773802963 (+1 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/A: creating writer at 1731773802963Flushing e301819a7ac81b4179f3ff5e1e4d8bad/A: appending metadata at 1731773802970 (+7 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/A: closing flushed file at 1731773802970Flushing e301819a7ac81b4179f3ff5e1e4d8bad/B: creating writer at 1731773803385 (+415 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/B: appending metadata at 1731773803389 (+4 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/B: closing flushed file at 1731773803389Flushing e301819a7ac81b4179f3ff5e1e4d8bad/C: creating writer at 1731773803803 (+414 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/C: appending metadata at 1731773803805 (+2 ms)Flushing e301819a7ac81b4179f3ff5e1e4d8bad/C: closing flushed file at 1731773803805Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c958cee: reopening flushed file at 1731773804213 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d31d0ef: reopening flushed file at 1731773804216 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@364e1bfe: reopening flushed file at 1731773804219 (+3 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=0 B/0 for e301819a7ac81b4179f3ff5e1e4d8bad in 1262ms, sequenceid=358, compaction requested=true at 1731773804223 (+4 ms)Writing region close event to WAL at 1731773804287 (+64 ms)Running coprocessor post-close hooks at 1731773804291 (+4 ms)Closed at 1731773804291 2024-11-16T16:16:44,294 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:44,295 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=e301819a7ac81b4179f3ff5e1e4d8bad, regionState=CLOSED 2024-11-16T16:16:44,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:44,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-16T16:16:44,301 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure e301819a7ac81b4179f3ff5e1e4d8bad, server=303fe01965e8,37355,1731773652566 in 1.4920 sec 2024-11-16T16:16:44,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-16T16:16:44,303 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e301819a7ac81b4179f3ff5e1e4d8bad, UNASSIGN in 1.4980 sec 2024-11-16T16:16:44,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-16T16:16:44,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5030 sec 2024-11-16T16:16:44,307 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773804307"}]},"ts":"1731773804307"} 2024-11-16T16:16:44,308 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:16:44,309 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:16:44,310 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-16T16:16:44,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-16T16:16:44,946 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:44,947 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:16:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:44,951 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T16:16:44,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:44,959 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:44,964 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/recovered.edits] 2024-11-16T16:16:44,968 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/1f823683568b4dbf90ad74f2e01a35d8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/1f823683568b4dbf90ad74f2e01a35d8 2024-11-16T16:16:44,971 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/4278954b53854184a2b1bb784d7591f4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/4278954b53854184a2b1bb784d7591f4 2024-11-16T16:16:44,974 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6fa21830d9a84acd81bd4f9a9636811a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/A/6fa21830d9a84acd81bd4f9a9636811a 2024-11-16T16:16:44,977 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/21532528d65e40398332230ce2f2a47e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/21532528d65e40398332230ce2f2a47e 2024-11-16T16:16:44,979 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/40a55d921e244186b96e37e88c7b1586 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/40a55d921e244186b96e37e88c7b1586 2024-11-16T16:16:44,980 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/61f726219edb4581972e89bfcc7b74e9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/B/61f726219edb4581972e89bfcc7b74e9 2024-11-16T16:16:44,983 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/a0d4f0c5cf7e4df591058d1f041a6a6d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/a0d4f0c5cf7e4df591058d1f041a6a6d 2024-11-16T16:16:44,984 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/e7baeaa7679641729845bec7c496db6d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/e7baeaa7679641729845bec7c496db6d 2024-11-16T16:16:44,986 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f29ff4a1314142fdafe7cb62627be966 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/C/f29ff4a1314142fdafe7cb62627be966 2024-11-16T16:16:44,989 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/recovered.edits/361.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad/recovered.edits/361.seqid 2024-11-16T16:16:44,989 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/e301819a7ac81b4179f3ff5e1e4d8bad 2024-11-16T16:16:44,989 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:16:44,992 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:44,995 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:16:44,997 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:16:44,998 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:44,998 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:16:44,998 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773804998"}]},"ts":"9223372036854775807"} 2024-11-16T16:16:45,001 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:16:45,001 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e301819a7ac81b4179f3ff5e1e4d8bad, NAME => 'TestAcidGuarantees,,1731773771555.e301819a7ac81b4179f3ff5e1e4d8bad.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:16:45,001 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:16:45,001 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773805001"}]},"ts":"9223372036854775807"} 2024-11-16T16:16:45,003 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:16:45,004 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:45,005 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-16T16:16:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-16T16:16:45,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:16:45,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:45,071 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248 (was 251), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=394 (was 510), ProcessCount=11 (was 11), AvailableMemoryMB=3831 (was 3826) - AvailableMemoryMB LEAK? - 2024-11-16T16:16:45,079 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=394, ProcessCount=11, AvailableMemoryMB=3830 2024-11-16T16:16:45,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:16:45,080 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:16:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:45,082 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T16:16:45,082 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:45,082 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-16T16:16:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:45,083 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T16:16:45,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742374_1550 (size=1061) 2024-11-16T16:16:45,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:45,493 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050 2024-11-16T16:16:45,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742375_1551 (size=53) 2024-11-16T16:16:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:45,902 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:16:45,902 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3818d7232c0ec9914127cf9e2f87f6dd, disabling compactions & flushes 2024-11-16T16:16:45,902 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:45,902 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:45,902 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. after waiting 0 ms 2024-11-16T16:16:45,902 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:45,903 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:45,903 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3818d7232c0ec9914127cf9e2f87f6dd: Waiting for close lock at 1731773805902Disabling compacts and flushes for region at 1731773805902Disabling writes for close at 1731773805902Writing region close event to WAL at 1731773805903 (+1 ms)Closed at 1731773805903 2024-11-16T16:16:45,905 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T16:16:45,905 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731773805905"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731773805905"}]},"ts":"1731773805905"} 2024-11-16T16:16:45,908 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T16:16:45,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T16:16:45,909 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773805909"}]},"ts":"1731773805909"} 2024-11-16T16:16:45,911 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-16T16:16:45,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, ASSIGN}] 2024-11-16T16:16:45,912 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, ASSIGN 2024-11-16T16:16:45,913 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, ASSIGN; state=OFFLINE, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=false 2024-11-16T16:16:46,064 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:46,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, ASSIGN because future has completed 2024-11-16T16:16:46,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:46,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:46,230 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:46,230 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:16:46,230 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,231 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:16:46,231 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,231 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,233 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,235 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:46,235 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName A 2024-11-16T16:16:46,235 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:46,236 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:46,236 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,238 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:46,238 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName B 2024-11-16T16:16:46,238 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:46,239 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:46,239 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,240 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:46,240 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName C 2024-11-16T16:16:46,240 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:46,241 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:46,241 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:46,241 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,242 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,242 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,244 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,244 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,244 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:16:46,246 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,248 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T16:16:46,249 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 3818d7232c0ec9914127cf9e2f87f6dd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74002231, jitterRate=0.10271917283535004}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:16:46,249 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:46,250 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 3818d7232c0ec9914127cf9e2f87f6dd: Running coprocessor pre-open hook at 1731773806231Writing region info on filesystem at 1731773806231Initializing all the Stores at 1731773806232 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773806232Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773806233 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773806233Cleaning up temporary data from old regions at 1731773806244 (+11 ms)Running coprocessor post-open hooks at 1731773806249 (+5 ms)Region opened successfully at 1731773806249 2024-11-16T16:16:46,250 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., pid=148, masterSystemTime=1731773806225 2024-11-16T16:16:46,252 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:46,253 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:46,253 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=OPEN, openSeqNum=2, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:46,255 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:46,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-16T16:16:46,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 in 186 msec 2024-11-16T16:16:46,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-16T16:16:46,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, ASSIGN in 346 msec 2024-11-16T16:16:46,259 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T16:16:46,259 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773806259"}]},"ts":"1731773806259"} 2024-11-16T16:16:46,260 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-16T16:16:46,260 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T16:16:46,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-16T16:16:47,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-16T16:16:47,227 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:47,229 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a04de68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:47,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:47,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:47,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a15889e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:47,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:47,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:47,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66a35303, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:47,237 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:47,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:47,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:47,240 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T16:16:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 303fe01965e8,45907,1731773651719 2024-11-16T16:16:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1e906d14 2024-11-16T16:16:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T16:16:47,242 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T16:16:47,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-16T16:16:47,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T16:16:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-16T16:16:47,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742376_1552 (size=1097) 2024-11-16T16:16:47,660 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-16T16:16:47,660 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-16T16:16:47,666 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:16:47,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE}] 2024-11-16T16:16:47,669 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE 2024-11-16T16:16:47,670 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:47,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE because future has completed 2024-11-16T16:16:47,671 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:16:47,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:47,825 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:47,825 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:16:47,825 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 3818d7232c0ec9914127cf9e2f87f6dd, disabling compactions & flushes 2024-11-16T16:16:47,825 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:47,825 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:47,825 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. after waiting 0 ms 2024-11-16T16:16:47,825 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:47,834 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-16T16:16:47,835 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:47,835 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 3818d7232c0ec9914127cf9e2f87f6dd: Waiting for close lock at 1731773807825Running coprocessor pre-close hooks at 1731773807825Disabling compacts and flushes for region at 1731773807825Disabling writes for close at 1731773807825Writing region close event to WAL at 1731773807826 (+1 ms)Running coprocessor post-close hooks at 1731773807835 (+9 ms)Closed at 1731773807835 2024-11-16T16:16:47,835 WARN [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 3818d7232c0ec9914127cf9e2f87f6dd to self. 2024-11-16T16:16:47,837 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:47,838 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=CLOSED 2024-11-16T16:16:47,840 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:47,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-16T16:16:47,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 in 170 msec 2024-11-16T16:16:47,843 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE; state=CLOSED, location=303fe01965e8,37355,1731773652566; forceNewPlan=false, retain=true 2024-11-16T16:16:47,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=OPENING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:47,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE because future has completed 2024-11-16T16:16:47,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:16:48,152 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,153 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} 2024-11-16T16:16:48,153 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,154 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T16:16:48,154 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,154 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,156 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,157 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:48,157 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName A 2024-11-16T16:16:48,159 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:48,159 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:48,160 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,160 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:48,160 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName B 2024-11-16T16:16:48,161 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:48,161 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:48,161 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,162 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-16T16:16:48,162 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3818d7232c0ec9914127cf9e2f87f6dd columnFamilyName C 2024-11-16T16:16:48,162 DEBUG [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:48,163 INFO [StoreOpener-3818d7232c0ec9914127cf9e2f87f6dd-1 {}] regionserver.HStore(327): Store=3818d7232c0ec9914127cf9e2f87f6dd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T16:16:48,163 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,163 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,164 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,165 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,166 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,166 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,166 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-16T16:16:48,167 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,167 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 3818d7232c0ec9914127cf9e2f87f6dd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69247008, jitterRate=0.0318608283996582}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-16T16:16:48,168 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,168 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 3818d7232c0ec9914127cf9e2f87f6dd: Running coprocessor pre-open hook at 1731773808154Writing region info on filesystem at 1731773808154Initializing all the Stores at 1731773808155 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773808155Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773808156 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731773808156Cleaning up temporary data from old regions at 1731773808166 (+10 ms)Running coprocessor post-open hooks at 1731773808168 (+2 ms)Region opened successfully at 1731773808168 2024-11-16T16:16:48,169 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., pid=153, masterSystemTime=1731773808146 2024-11-16T16:16:48,170 DEBUG [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,170 INFO [RS_OPEN_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,170 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=OPEN, openSeqNum=5, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:16:48,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-16T16:16:48,173 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 in 178 msec 2024-11-16T16:16:48,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-16T16:16:48,175 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, REOPEN/MOVE in 505 msec 2024-11-16T16:16:48,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-16T16:16:48,176 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-16T16:16:48,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 933 msec 2024-11-16T16:16:48,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-16T16:16:48,178 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:48,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53954261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,179 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,180 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ee5ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be3103f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2aa61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,182 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f07b7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,183 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41706, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344127fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,184 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca6c80d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,184 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,184 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,185 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19cd88c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,186 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efc1ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14afc90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,187 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,187 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,187 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@234324e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,189 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,189 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aae18f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,191 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@301549f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,191 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,192 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c96417e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,193 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99b73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,193 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,193 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,194 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6b2e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,195 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414ab9ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,197 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@261f7c95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2714d6d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,201 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f3da21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,202 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ddf3acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,204 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403c5e3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,206 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d363ec1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 303fe01965e8,45907,-1 for getting cluster id 2024-11-16T16:16:48,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T16:16:48,208 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd498f81b-52a7-4718-ba95-d95378acab0e' 2024-11-16T16:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T16:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d498f81b-52a7-4718-ba95-d95378acab0e" 2024-11-16T16:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a568200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [303fe01965e8,45907,-1] 2024-11-16T16:16:48,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T16:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:16:48,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T16:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2dc371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T16:16:48,210 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,210 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,210 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:48,210 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,210 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,211 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,211 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,212 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,213 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:48,213 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,213 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T16:16:48,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=303fe01965e8,37355,1731773652566, seqNum=-1] 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,214 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T16:16:48,214 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T16:16:48,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,216 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:48,216 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:48,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,218 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,219 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5] 2024-11-16T16:16:48,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773808235, max timestamp=1731773808239Num uniques 859; 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773808235, max timestamp=1731773808239Num uniques 859; 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773808235, max timestamp=1731773808239Num uniques 859; 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:48,242 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:16:48,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcaa06fb8317484a97c40b79b667fc50_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773808221/Put/seqid=0 2024-11-16T16:16:48,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742377_1553 (size=14594) 2024-11-16T16:16:48,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773868341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773868341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773868341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773868344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773868345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773868455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773868455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773868455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773868455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773868455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:48,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:48,654 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116fcaa06fb8317484a97c40b79b667fc50_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcaa06fb8317484a97c40b79b667fc50_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:48,655 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9dfb455ae012466b97c9266c51e8e330, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:48,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9dfb455ae012466b97c9266c51e8e330 is 175, key is test_row_0/A:col10/1731773808221/Put/seqid=0 2024-11-16T16:16:48,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742378_1554 (size=39549) 2024-11-16T16:16:48,658 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9dfb455ae012466b97c9266c51e8e330 2024-11-16T16:16:48,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773868664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773868665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773868665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773868665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773868665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/3baee1edf02740da8efb4ca44f988cd7 is 50, key is test_row_0/B:col10/1731773808221/Put/seqid=0 2024-11-16T16:16:48,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742379_1555 (size=12001) 2024-11-16T16:16:48,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:48,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773868975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773868975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773868976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773868976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:48,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:48,978 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:48,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:48,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773868977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/3baee1edf02740da8efb4ca44f988cd7 2024-11-16T16:16:49,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/805f99d4c8a54d479d73845fa3c70eb5 is 50, key is test_row_0/C:col10/1731773808221/Put/seqid=0 2024-11-16T16:16:49,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742380_1556 (size=12001) 2024-11-16T16:16:49,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:49,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:49,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773869484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773869485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773869485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773869485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773869485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:49,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/805f99d4c8a54d479d73845fa3c70eb5 2024-11-16T16:16:49,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9dfb455ae012466b97c9266c51e8e330 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330 2024-11-16T16:16:49,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330, entries=200, sequenceid=14, filesize=38.6 K 2024-11-16T16:16:49,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/3baee1edf02740da8efb4ca44f988cd7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7 2024-11-16T16:16:49,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T16:16:49,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/805f99d4c8a54d479d73845fa3c70eb5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5 2024-11-16T16:16:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5, entries=150, sequenceid=14, filesize=11.7 K 2024-11-16T16:16:49,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3818d7232c0ec9914127cf9e2f87f6dd in 1292ms, sequenceid=14, compaction requested=false 2024-11-16T16:16:49,514 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-16T16:16:49,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:49,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-16T16:16:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:49,588 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:49,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:49,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160a15cd87bc5a458b9de80ed2ff2913a5_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773808340/Put/seqid=0 2024-11-16T16:16:49,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742381_1557 (size=12154) 2024-11-16T16:16:49,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:49,600 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160a15cd87bc5a458b9de80ed2ff2913a5_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a15cd87bc5a458b9de80ed2ff2913a5_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf2afe4f45004a4db6630161553cbd05, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:49,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf2afe4f45004a4db6630161553cbd05 is 175, key is test_row_0/A:col10/1731773808340/Put/seqid=0 2024-11-16T16:16:49,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742382_1558 (size=30955) 2024-11-16T16:16:49,604 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf2afe4f45004a4db6630161553cbd05 2024-11-16T16:16:49,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/38924e94a3944f5db89794ecbff709e0 is 50, key is test_row_0/B:col10/1731773808340/Put/seqid=0 2024-11-16T16:16:49,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742383_1559 (size=12001) 2024-11-16T16:16:49,812 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T16:16:50,017 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/38924e94a3944f5db89794ecbff709e0 2024-11-16T16:16:50,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e1b97a96c7624a0b90aeff13f5de4082 is 50, key is test_row_0/C:col10/1731773808340/Put/seqid=0 2024-11-16T16:16:50,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742384_1560 (size=12001) 2024-11-16T16:16:50,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:50,430 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e1b97a96c7624a0b90aeff13f5de4082 2024-11-16T16:16:50,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf2afe4f45004a4db6630161553cbd05 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05 2024-11-16T16:16:50,437 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05, entries=150, sequenceid=51, filesize=30.2 K 2024-11-16T16:16:50,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/38924e94a3944f5db89794ecbff709e0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0 2024-11-16T16:16:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,440 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T16:16:50,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e1b97a96c7624a0b90aeff13f5de4082 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,443 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082, entries=150, sequenceid=51, filesize=11.7 K 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for 3818d7232c0ec9914127cf9e2f87f6dd in 856ms, sequenceid=51, compaction requested=false 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2370 sec 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2450 sec 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:50,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:50,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:50,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168cb3943dcc0847e7b2be5b842c630ab9_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:50,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742385_1561 (size=21918) 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773810540, max timestamp=1731773810546Num uniques 859; 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773810540, max timestamp=1731773810546Num uniques 859; 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773810540, max timestamp=1731773810546Num uniques 859; 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:50,549 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:50,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773870571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773870572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773870573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773870584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773870584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773870695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773870695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773870695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773870695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773870695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,698 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773870905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773870906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773870906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:50,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:50,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168cb3943dcc0847e7b2be5b842c630ab9_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168cb3943dcc0847e7b2be5b842c630ab9_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:50,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4f9c4465c0574f2cad72de575ab3f739, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:50,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4f9c4465c0574f2cad72de575ab3f739 is 175, key is test_row_0/A:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742386_1562 (size=65323) 2024-11-16T16:16:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773871215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773871215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773871215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773871215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773871215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,219 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4f9c4465c0574f2cad72de575ab3f739 2024-11-16T16:16:51,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/9c97417d8c014d4d9d524299811ae39e is 50, key is test_row_0/B:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:51,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742387_1563 (size=12001) 2024-11-16T16:16:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773871725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773871725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773871725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773871725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773871725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:51,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:51,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/9c97417d8c014d4d9d524299811ae39e 2024-11-16T16:16:51,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/66e5bf1e6ab848078ebef8224c8f0fa5 is 50, key is test_row_0/C:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742388_1564 (size=12001) 2024-11-16T16:16:52,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/66e5bf1e6ab848078ebef8224c8f0fa5 2024-11-16T16:16:52,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4f9c4465c0574f2cad72de575ab3f739 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739 2024-11-16T16:16:52,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739, entries=350, sequenceid=61, filesize=63.8 K 2024-11-16T16:16:52,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/9c97417d8c014d4d9d524299811ae39e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e 2024-11-16T16:16:52,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:16:52,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/66e5bf1e6ab848078ebef8224c8f0fa5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5 2024-11-16T16:16:52,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5, entries=150, sequenceid=61, filesize=11.7 K 2024-11-16T16:16:52,175 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3818d7232c0ec9914127cf9e2f87f6dd in 1655ms, sequenceid=61, compaction requested=true 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:52,175 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:52,175 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:52,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:52,176 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:52,176 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:16:52,176 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:52,176 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.2 K 2024-11-16T16:16:52,176 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:52,176 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:16:52,176 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:52,177 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=132.6 K 2024-11-16T16:16:52,177 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739] 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 3baee1edf02740da8efb4ca44f988cd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731773808220 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dfb455ae012466b97c9266c51e8e330, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731773808219 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 38924e94a3944f5db89794ecbff709e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773808283 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf2afe4f45004a4db6630161553cbd05, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773808283 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f9c4465c0574f2cad72de575ab3f739, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810507 2024-11-16T16:16:52,177 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c97417d8c014d4d9d524299811ae39e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810509 2024-11-16T16:16:52,181 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:52,182 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:52,183 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116be323eee215d49eb9f99013074a29fda_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:52,183 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/70ba82ea7b924bc2819464116b10ef95 is 50, key is test_row_0/B:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:52,185 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116be323eee215d49eb9f99013074a29fda_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:52,185 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116be323eee215d49eb9f99013074a29fda_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:52,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742389_1565 (size=12104) 2024-11-16T16:16:52,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742390_1566 (size=4469) 2024-11-16T16:16:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-16T16:16:52,375 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:52,375 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-16T16:16:52,376 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:52,377 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:52,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:52,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-16T16:16:52,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:52,530 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:52,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dd887cc27639457198d9ebf5f0caf312_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773810569/Put/seqid=0 2024-11-16T16:16:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742391_1567 (size=12154) 2024-11-16T16:16:52,589 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:52,589 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4feb80f2cdb142b5aa8aea8728826f79 is 175, key is test_row_0/A:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:52,591 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/70ba82ea7b924bc2819464116b10ef95 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/70ba82ea7b924bc2819464116b10ef95 2024-11-16T16:16:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742392_1568 (size=31058) 2024-11-16T16:16:52,596 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into 70ba82ea7b924bc2819464116b10ef95(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:52,596 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/4feb80f2cdb142b5aa8aea8728826f79 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79 2024-11-16T16:16:52,596 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:52,596 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=13, startTime=1731773812175; duration=0sec 2024-11-16T16:16:52,596 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:52,596 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:16:52,596 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:52,597 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:52,597 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:16:52,597 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:52,597 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.2 K 2024-11-16T16:16:52,597 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 805f99d4c8a54d479d73845fa3c70eb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731773808220 2024-11-16T16:16:52,597 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b97a96c7624a0b90aeff13f5de4082, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731773808283 2024-11-16T16:16:52,597 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 66e5bf1e6ab848078ebef8224c8f0fa5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810509 2024-11-16T16:16:52,607 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into 4feb80f2cdb142b5aa8aea8728826f79(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:52,607 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:52,607 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=13, startTime=1731773812175; duration=0sec 2024-11-16T16:16:52,608 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:52,608 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:16:52,623 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#488 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:52,623 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/6f439ca5f0814a8db0c7f7a543669782 is 50, key is test_row_0/C:col10/1731773810509/Put/seqid=0 2024-11-16T16:16:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742393_1569 (size=12104) 2024-11-16T16:16:52,633 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/6f439ca5f0814a8db0c7f7a543669782 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6f439ca5f0814a8db0c7f7a543669782 2024-11-16T16:16:52,637 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into 6f439ca5f0814a8db0c7f7a543669782(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:52,637 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:52,637 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=13, startTime=1731773812175; duration=0sec 2024-11-16T16:16:52,637 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:52,637 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:16:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:52,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:52,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:52,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773872745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773872745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,749 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:52,749 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773872747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773872747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,752 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773872749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773872855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773872855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773872855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:52,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:52,942 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116dd887cc27639457198d9ebf5f0caf312_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dd887cc27639457198d9ebf5f0caf312_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/82e2fd870cc148328d679dd86e4af6ae, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:52,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/82e2fd870cc148328d679dd86e4af6ae is 175, key is test_row_0/A:col10/1731773810569/Put/seqid=0 2024-11-16T16:16:52,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742394_1570 (size=30955) 2024-11-16T16:16:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773873065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773873065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773873065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,069 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,349 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/82e2fd870cc148328d679dd86e4af6ae 2024-11-16T16:16:53,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0aae01e9901d4e1b8bf19373358b3af3 is 50, key is test_row_0/B:col10/1731773810569/Put/seqid=0 2024-11-16T16:16:53,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742395_1571 (size=12001) 2024-11-16T16:16:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773873375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773873375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773873375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:53,765 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0aae01e9901d4e1b8bf19373358b3af3 2024-11-16T16:16:53,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/74c878559b6a4f2c80a2551a3f7a9a27 is 50, key is test_row_0/C:col10/1731773810569/Put/seqid=0 2024-11-16T16:16:53,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742396_1572 (size=12001) 2024-11-16T16:16:53,778 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/74c878559b6a4f2c80a2551a3f7a9a27 2024-11-16T16:16:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/82e2fd870cc148328d679dd86e4af6ae as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae 2024-11-16T16:16:53,787 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae, entries=150, sequenceid=97, filesize=30.2 K 2024-11-16T16:16:53,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0aae01e9901d4e1b8bf19373358b3af3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3 2024-11-16T16:16:53,791 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3, entries=150, sequenceid=97, filesize=11.7 K 2024-11-16T16:16:53,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/74c878559b6a4f2c80a2551a3f7a9a27 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27 2024-11-16T16:16:53,795 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27, entries=150, sequenceid=97, filesize=11.7 K 2024-11-16T16:16:53,796 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 3818d7232c0ec9914127cf9e2f87f6dd in 1267ms, sequenceid=97, compaction requested=false 2024-11-16T16:16:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-16T16:16:53,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-16T16:16:53,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-16T16:16:53,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4220 sec 2024-11-16T16:16:53,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4270 sec 2024-11-16T16:16:53,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:53,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:53,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:53,917 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167cb70eebe6274952a2bf2ee5ff3459ff_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:53,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773873933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773873937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:53,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773873941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:53,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:53,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:53,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742397_1573 (size=12154) 2024-11-16T16:16:53,971 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:53,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411167cb70eebe6274952a2bf2ee5ff3459ff_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cb70eebe6274952a2bf2ee5ff3459ff_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:53,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:53,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc is 175, key is test_row_0/A:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:53,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742398_1574 (size=30955) 2024-11-16T16:16:53,995 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=112, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc 2024-11-16T16:16:54,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/5e743b9221eb4336a73b09b7197d065d is 50, key is test_row_0/B:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:54,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742399_1575 (size=12001) 2024-11-16T16:16:54,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773874045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773874055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773874055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773874255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773874264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773874264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/5e743b9221eb4336a73b09b7197d065d 2024-11-16T16:16:54,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/20b25686069049ffa2ed6ed236fbca27 is 50, key is test_row_0/C:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:54,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742400_1576 (size=12001) 2024-11-16T16:16:54,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-16T16:16:54,526 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:54,526 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-16T16:16:54,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T16:16:54,529 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:54,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:54,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773874565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773874576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773874576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,581 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T16:16:54,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T16:16:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773874765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773874765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:54,770 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,771 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:54,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:54,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T16:16:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T16:16:54,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/20b25686069049ffa2ed6ed236fbca27 2024-11-16T16:16:54,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc 2024-11-16T16:16:54,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc, entries=150, sequenceid=112, filesize=30.2 K 2024-11-16T16:16:54,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/5e743b9221eb4336a73b09b7197d065d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d 2024-11-16T16:16:54,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T16:16:54,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/20b25686069049ffa2ed6ed236fbca27 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27 2024-11-16T16:16:54,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27, entries=150, sequenceid=112, filesize=11.7 K 2024-11-16T16:16:54,878 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3818d7232c0ec9914127cf9e2f87f6dd in 989ms, sequenceid=112, compaction requested=true 2024-11-16T16:16:54,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:54,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:16:54,879 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:16:54,879 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,879 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,880 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=90.8 K 2024-11-16T16:16:54,880 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/70ba82ea7b924bc2819464116b10ef95, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.3 K 2024-11-16T16:16:54,880 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc] 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ba82ea7b924bc2819464116b10ef95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810509 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4feb80f2cdb142b5aa8aea8728826f79, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810509 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aae01e9901d4e1b8bf19373358b3af3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731773810569 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82e2fd870cc148328d679dd86e4af6ae, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731773810569 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e743b9221eb4336a73b09b7197d065d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:54,880 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41c4dd93b9da4d9a983a5e2e7a3fe5fc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:54,884 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:54,886 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:54,886 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116bd1cd53003204120be8efa998ad7e5bc_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:54,886 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0b393cd9413a4dbb9ec18dd2b0b46dd6 is 50, key is test_row_0/B:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:54,888 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116bd1cd53003204120be8efa998ad7e5bc_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:54,888 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116bd1cd53003204120be8efa998ad7e5bc_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:54,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742401_1577 (size=12207) 2024-11-16T16:16:54,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742402_1578 (size=4469) 2024-11-16T16:16:54,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:54,994 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116716584e887f74dcb92367ea4e940a47c_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773813939/Put/seqid=0 2024-11-16T16:16:55,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742403_1579 (size=12304) 2024-11-16T16:16:55,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:55,042 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116716584e887f74dcb92367ea4e940a47c_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116716584e887f74dcb92367ea4e940a47c_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:55,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b0b1a48dd01246658b52b3a5cd84082f, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b0b1a48dd01246658b52b3a5cd84082f is 175, key is test_row_0/A:col10/1731773813939/Put/seqid=0 2024-11-16T16:16:55,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742404_1580 (size=31105) 2024-11-16T16:16:55,076 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b0b1a48dd01246658b52b3a5cd84082f 2024-11-16T16:16:55,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:55,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:55,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/f786a4998d4b40e9ad903f15d102674e is 50, key is test_row_0/B:col10/1731773813939/Put/seqid=0 2024-11-16T16:16:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742405_1581 (size=12151) 2024-11-16T16:16:55,104 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/f786a4998d4b40e9ad903f15d102674e 2024-11-16T16:16:55,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773875119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773875120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773875120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e3887d953aed4a2f8b0d045a6b532b5a is 50, key is test_row_0/C:col10/1731773813939/Put/seqid=0 2024-11-16T16:16:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742406_1582 (size=12151) 2024-11-16T16:16:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T16:16:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773875224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,230 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773875235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773875235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,307 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:55,307 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/add94e286e514f29a2cf45c61526d754 is 175, key is test_row_0/A:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:55,309 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0b393cd9413a4dbb9ec18dd2b0b46dd6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0b393cd9413a4dbb9ec18dd2b0b46dd6 2024-11-16T16:16:55,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742407_1583 (size=31161) 2024-11-16T16:16:55,314 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into 0b393cd9413a4dbb9ec18dd2b0b46dd6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:55,314 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:55,314 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=13, startTime=1731773814879; duration=0sec 2024-11-16T16:16:55,314 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:55,314 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:16:55,314 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:55,315 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:55,315 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:16:55,316 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,316 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6f439ca5f0814a8db0c7f7a543669782, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.3 K 2024-11-16T16:16:55,316 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f439ca5f0814a8db0c7f7a543669782, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731773810509 2024-11-16T16:16:55,316 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c878559b6a4f2c80a2551a3f7a9a27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731773810569 2024-11-16T16:16:55,316 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 20b25686069049ffa2ed6ed236fbca27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:55,317 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/add94e286e514f29a2cf45c61526d754 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754 2024-11-16T16:16:55,322 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into add94e286e514f29a2cf45c61526d754(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:55,322 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:55,322 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=13, startTime=1731773814878; duration=0sec 2024-11-16T16:16:55,322 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:55,322 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:16:55,324 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#499 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:55,325 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/a12af6c370084047a48c60efc0f46769 is 50, key is test_row_0/C:col10/1731773813887/Put/seqid=0 2024-11-16T16:16:55,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742408_1584 (size=12207) 2024-11-16T16:16:55,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773875435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773875445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773875445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,554 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e3887d953aed4a2f8b0d045a6b532b5a 2024-11-16T16:16:55,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b0b1a48dd01246658b52b3a5cd84082f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f 2024-11-16T16:16:55,569 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f, entries=150, sequenceid=132, filesize=30.4 K 2024-11-16T16:16:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/f786a4998d4b40e9ad903f15d102674e as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e 2024-11-16T16:16:55,574 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T16:16:55,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/e3887d953aed4a2f8b0d045a6b532b5a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a 2024-11-16T16:16:55,579 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a, entries=150, sequenceid=132, filesize=11.9 K 2024-11-16T16:16:55,580 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 3818d7232c0ec9914127cf9e2f87f6dd in 586ms, sequenceid=132, compaction requested=false 2024-11-16T16:16:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-16T16:16:55,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-16T16:16:55,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-16T16:16:55,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-11-16T16:16:55,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0590 sec 2024-11-16T16:16:55,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-16T16:16:55,665 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:55,665 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-16T16:16:55,667 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:55,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:55,668 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:55,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:55,738 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/a12af6c370084047a48c60efc0f46769 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/a12af6c370084047a48c60efc0f46769 2024-11-16T16:16:55,748 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into a12af6c370084047a48c60efc0f46769(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:55,748 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:55,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:55,749 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=13, startTime=1731773814879; duration=0sec 2024-11-16T16:16:55,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-16T16:16:55,749 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:55,749 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:55,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:55,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:55,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111609b07bba518545248c1d583991b9c198_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:55,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773815765, max timestamp=1731773815792Num uniques 859; 2024-11-16T16:16:55,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773815765, max timestamp=1731773815792Num uniques 859; 2024-11-16T16:16:55,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:55,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:55,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731773815765, max timestamp=1731773815792Num uniques 859; 2024-11-16T16:16:55,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-16T16:16:55,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742409_1585 (size=12304) 2024-11-16T16:16:55,814 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:55,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111609b07bba518545248c1d583991b9c198_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609b07bba518545248c1d583991b9c198_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:55,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/f536d41e1a884559a7a978b434a54a5d, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/f536d41e1a884559a7a978b434a54a5d is 175, key is test_row_0/A:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:55,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742410_1586 (size=31105) 2024-11-16T16:16:55,831 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=148, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/f536d41e1a884559a7a978b434a54a5d 2024-11-16T16:16:55,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773875827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773875830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773875831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/cb23ad01fe9d48148a750c80ee34e2b9 is 50, key is test_row_0/B:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:55,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742411_1587 (size=12151) 2024-11-16T16:16:55,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/cb23ad01fe9d48148a750c80ee34e2b9 2024-11-16T16:16:55,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/617e9f5b32354bef8281a6e9b4252437 is 50, key is test_row_0/C:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742412_1588 (size=12151) 2024-11-16T16:16:55,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/617e9f5b32354bef8281a6e9b4252437 2024-11-16T16:16:55,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/f536d41e1a884559a7a978b434a54a5d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d 2024-11-16T16:16:55,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d, entries=150, sequenceid=148, filesize=30.4 K 2024-11-16T16:16:55,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/cb23ad01fe9d48148a750c80ee34e2b9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9 2024-11-16T16:16:55,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T16:16:55,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/617e9f5b32354bef8281a6e9b4252437 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437 2024-11-16T16:16:55,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437, entries=150, sequenceid=148, filesize=11.9 K 2024-11-16T16:16:55,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 3818d7232c0ec9914127cf9e2f87f6dd in 163ms, sequenceid=148, compaction requested=true 2024-11-16T16:16:55,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:55,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:55,913 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:55,913 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:55,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:55,913 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93371 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:55,913 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:55,914 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:16:55,914 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:16:55,914 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,914 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,914 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0b393cd9413a4dbb9ec18dd2b0b46dd6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.7 K 2024-11-16T16:16:55,914 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=91.2 K 2024-11-16T16:16:55,914 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,914 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d] 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting add94e286e514f29a2cf45c61526d754, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b393cd9413a4dbb9ec18dd2b0b46dd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f786a4998d4b40e9ad903f15d102674e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731773813930 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0b1a48dd01246658b52b3a5cd84082f, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731773813930 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting cb23ad01fe9d48148a750c80ee34e2b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:55,916 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting f536d41e1a884559a7a978b434a54a5d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:55,923 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,930 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:55,931 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/ee1559fbcd154080856b1e623528b7b8 is 50, key is test_row_0/B:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:55,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-11-16T16:16:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:55,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:55,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:55,941 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116ffc1e4594d6c437aa52ab24bc37b6808_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,943 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116ffc1e4594d6c437aa52ab24bc37b6808_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,943 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ffc1e4594d6c437aa52ab24bc37b6808_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:55,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:55,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:55,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773875968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773875969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:55,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773875975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:55,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:55,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:55,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742413_1589 (size=12459) 2024-11-16T16:16:55,999 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/ee1559fbcd154080856b1e623528b7b8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/ee1559fbcd154080856b1e623528b7b8 2024-11-16T16:16:56,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d34e47b55e434e1598887e1238a4ca93_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773815828/Put/seqid=0 2024-11-16T16:16:56,004 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into ee1559fbcd154080856b1e623528b7b8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:56,004 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:56,004 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=13, startTime=1731773815913; duration=0sec 2024-11-16T16:16:56,004 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:56,004 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:16:56,004 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:16:56,005 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:16:56,005 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:16:56,005 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,005 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/a12af6c370084047a48c60efc0f46769, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=35.7 K 2024-11-16T16:16:56,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a12af6c370084047a48c60efc0f46769, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731773812741 2024-11-16T16:16:56,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting e3887d953aed4a2f8b0d045a6b532b5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731773813930 2024-11-16T16:16:56,006 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 617e9f5b32354bef8281a6e9b4252437, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:56,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742414_1590 (size=4469) 2024-11-16T16:16:56,014 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#503 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:56,015 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/6942ef0f038d49b6b9b5ff58fdb5b2c8 is 175, key is test_row_0/A:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:56,023 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#506 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:56,023 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/7df355a9a8eb4a2aa700d232947aa42a is 50, key is test_row_0/C:col10/1731773815098/Put/seqid=0 2024-11-16T16:16:56,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742415_1591 (size=14794) 2024-11-16T16:16:56,034 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:56,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116d34e47b55e434e1598887e1238a4ca93_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d34e47b55e434e1598887e1238a4ca93_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:56,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/3c40ac7dba80441f80b191fe9ffabbad, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:56,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/3c40ac7dba80441f80b191fe9ffabbad is 175, key is test_row_0/A:col10/1731773815828/Put/seqid=0 2024-11-16T16:16:56,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742416_1592 (size=31413) 2024-11-16T16:16:56,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742417_1593 (size=12459) 2024-11-16T16:16:56,066 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/7df355a9a8eb4a2aa700d232947aa42a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/7df355a9a8eb4a2aa700d232947aa42a 2024-11-16T16:16:56,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742418_1594 (size=39749) 2024-11-16T16:16:56,071 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/3c40ac7dba80441f80b191fe9ffabbad 2024-11-16T16:16:56,072 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into 7df355a9a8eb4a2aa700d232947aa42a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:56,072 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:56,072 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=13, startTime=1731773815913; duration=0sec 2024-11-16T16:16:56,072 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:56,072 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:16:56,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b1faffc3ca774129a3cc4c26ed2dd390 is 50, key is test_row_0/B:col10/1731773815828/Put/seqid=0 2024-11-16T16:16:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773876085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773876085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773876085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,087 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742419_1595 (size=12151) 2024-11-16T16:16:56,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773876295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773876295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773876295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,454 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/6942ef0f038d49b6b9b5ff58fdb5b2c8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8 2024-11-16T16:16:56,459 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into 6942ef0f038d49b6b9b5ff58fdb5b2c8(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:56,459 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:56,459 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=13, startTime=1731773815912; duration=0sec 2024-11-16T16:16:56,459 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:56,459 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:16:56,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b1faffc3ca774129a3cc4c26ed2dd390 2024-11-16T16:16:56,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/930ef3868a574774ba1a259dfa458e83 is 50, key is test_row_0/C:col10/1731773815828/Put/seqid=0 2024-11-16T16:16:56,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742420_1596 (size=12151) 2024-11-16T16:16:56,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773876605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773876605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773876605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:56,609 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:56,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:56,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:56,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:16:56,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/930ef3868a574774ba1a259dfa458e83 2024-11-16T16:16:56,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/3c40ac7dba80441f80b191fe9ffabbad as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad 2024-11-16T16:16:56,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad, entries=200, sequenceid=182, filesize=38.8 K 2024-11-16T16:16:56,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b1faffc3ca774129a3cc4c26ed2dd390 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390 2024-11-16T16:16:56,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T16:16:56,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/930ef3868a574774ba1a259dfa458e83 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83 2024-11-16T16:16:56,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83, entries=150, sequenceid=182, filesize=11.9 K 2024-11-16T16:16:56,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 3818d7232c0ec9914127cf9e2f87f6dd in 988ms, sequenceid=182, compaction requested=false 2024-11-16T16:16:56,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:57,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-16T16:16:57,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:57,046 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:57,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:16:57,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111695f2e29762784a9db8faa2898d78b985_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773815954/Put/seqid=0 2024-11-16T16:16:57,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742421_1597 (size=12304) 2024-11-16T16:16:57,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:57,082 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111695f2e29762784a9db8faa2898d78b985_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111695f2e29762784a9db8faa2898d78b985_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:57,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/c8b7310250d54bebac723950db69d3ba, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:57,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/c8b7310250d54bebac723950db69d3ba is 175, key is test_row_0/A:col10/1731773815954/Put/seqid=0 2024-11-16T16:16:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:57,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:16:57,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742422_1598 (size=31105) 2024-11-16T16:16:57,124 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/c8b7310250d54bebac723950db69d3ba 2024-11-16T16:16:57,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b0244856f7854f58b6f310c7c537de85 is 50, key is test_row_0/B:col10/1731773815954/Put/seqid=0 2024-11-16T16:16:57,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742423_1599 (size=12151) 2024-11-16T16:16:57,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773877148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773877150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773877154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,268 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773877265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773877475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773877475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773877475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,544 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b0244856f7854f58b6f310c7c537de85 2024-11-16T16:16:57,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/06a3cbc73a20487d80973937a9c0ee58 is 50, key is test_row_0/C:col10/1731773815954/Put/seqid=0 2024-11-16T16:16:57,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742424_1600 (size=12151) 2024-11-16T16:16:57,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773877784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,787 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773877785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773877785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:57,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:57,961 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/06a3cbc73a20487d80973937a9c0ee58 2024-11-16T16:16:57,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/c8b7310250d54bebac723950db69d3ba as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba 2024-11-16T16:16:57,969 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba, entries=150, sequenceid=197, filesize=30.4 K 2024-11-16T16:16:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/b0244856f7854f58b6f310c7c537de85 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85 2024-11-16T16:16:57,973 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T16:16:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/06a3cbc73a20487d80973937a9c0ee58 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58 2024-11-16T16:16:57,977 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58, entries=150, sequenceid=197, filesize=11.9 K 2024-11-16T16:16:57,978 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 3818d7232c0ec9914127cf9e2f87f6dd in 931ms, sequenceid=197, compaction requested=true 2024-11-16T16:16:57,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:57,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:57,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-16T16:16:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-16T16:16:57,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-16T16:16:57,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3110 sec 2024-11-16T16:16:57,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.3150 sec 2024-11-16T16:16:58,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-16T16:16:58,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:16:58,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:58,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:16:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:16:58,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:16:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:58,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773878321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773878325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773878326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c1cbc85993d549148676dbc1fb0b5cb9_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:58,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742425_1601 (size=12304) 2024-11-16T16:16:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773878435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773878645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773878645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773878645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55370 deadline: 1731773878784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,787 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_2' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,788 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55348 deadline: 1731773878794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,796 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,810 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:16:58,813 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116c1cbc85993d549148676dbc1fb0b5cb9_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1cbc85993d549148676dbc1fb0b5cb9_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:16:58,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/44eac1f2bcac4554a58d81096468b314, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:58,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/44eac1f2bcac4554a58d81096468b314 is 175, key is test_row_0/A:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:58,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742426_1602 (size=31105) 2024-11-16T16:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773878955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773878955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,958 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773878955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:58,959 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:59,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/44eac1f2bcac4554a58d81096468b314 2024-11-16T16:16:59,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/7e5965232f46415ea8ffdb974bfb4476 is 50, key is test_row_0/B:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:59,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742427_1603 (size=12151) 2024-11-16T16:16:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:59,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773879465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:59,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773879465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:59,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:16:59,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773879465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:59,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:16:59,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/7e5965232f46415ea8ffdb974bfb4476 2024-11-16T16:16:59,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/bcad65d051a9479bb43582b69fdff365 is 50, key is test_row_0/C:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:59,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742428_1604 (size=12151) 2024-11-16T16:16:59,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/bcad65d051a9479bb43582b69fdff365 2024-11-16T16:16:59,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/44eac1f2bcac4554a58d81096468b314 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314 2024-11-16T16:16:59,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314, entries=150, sequenceid=218, filesize=30.4 K 2024-11-16T16:16:59,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/7e5965232f46415ea8ffdb974bfb4476 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476 2024-11-16T16:16:59,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T16:16:59,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/bcad65d051a9479bb43582b69fdff365 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365 2024-11-16T16:16:59,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365, entries=150, sequenceid=218, filesize=11.9 K 2024-11-16T16:16:59,690 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 3818d7232c0ec9914127cf9e2f87f6dd in 1391ms, sequenceid=218, compaction requested=true 2024-11-16T16:16:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:16:59,690 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:59,690 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:16:59,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:59,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:16:59,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:59,691 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:59,691 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:16:59,691 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:59,692 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=130.2 K 2024-11-16T16:16:59,692 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314] 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:16:59,692 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:59,692 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/ee1559fbcd154080856b1e623528b7b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=47.8 K 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6942ef0f038d49b6b9b5ff58fdb5b2c8, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ee1559fbcd154080856b1e623528b7b8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:59,692 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c40ac7dba80441f80b191fe9ffabbad, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773815811 2024-11-16T16:16:59,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b1faffc3ca774129a3cc4c26ed2dd390, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773815811 2024-11-16T16:16:59,693 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b7310250d54bebac723950db69d3ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731773815954 2024-11-16T16:16:59,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting b0244856f7854f58b6f310c7c537de85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731773815954 2024-11-16T16:16:59,693 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44eac1f2bcac4554a58d81096468b314, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:16:59,693 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e5965232f46415ea8ffdb974bfb4476, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:16:59,715 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:59,715 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/a621cd9056de434cbc2a84e65f9affa2 is 50, key is test_row_0/B:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:59,718 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:59,722 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111615907c3b34414175a274cb7d0446e618_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:59,724 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111615907c3b34414175a274cb7d0446e618_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:59,724 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111615907c3b34414175a274cb7d0446e618_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:16:59,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742429_1605 (size=12595) 2024-11-16T16:16:59,750 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/a621cd9056de434cbc2a84e65f9affa2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a621cd9056de434cbc2a84e65f9affa2 2024-11-16T16:16:59,755 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into a621cd9056de434cbc2a84e65f9affa2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:59,755 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:59,755 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=12, startTime=1731773819690; duration=0sec 2024-11-16T16:16:59,755 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:16:59,755 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:16:59,755 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:16:59,756 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:16:59,756 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:16:59,757 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:16:59,757 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/7df355a9a8eb4a2aa700d232947aa42a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=47.8 K 2024-11-16T16:16:59,758 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 7df355a9a8eb4a2aa700d232947aa42a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731773815098 2024-11-16T16:16:59,758 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 930ef3868a574774ba1a259dfa458e83, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731773815811 2024-11-16T16:16:59,758 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a3cbc73a20487d80973937a9c0ee58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1731773815954 2024-11-16T16:16:59,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742430_1606 (size=4469) 2024-11-16T16:16:59,759 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting bcad65d051a9479bb43582b69fdff365, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:16:59,761 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#516 average throughput is 0.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:59,762 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006 is 175, key is test_row_0/A:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:59,782 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:16:59,783 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/ebf73770c7bd42bf8bb6e433e79c860f is 50, key is test_row_0/C:col10/1731773818298/Put/seqid=0 2024-11-16T16:16:59,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742431_1607 (size=31549) 2024-11-16T16:16:59,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742432_1608 (size=12595) 2024-11-16T16:16:59,793 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006 2024-11-16T16:16:59,796 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into 1e7f0b11d2fe4a7aa8e4c1aaf0d43006(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:16:59,796 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:16:59,796 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=12, startTime=1731773819690; duration=0sec 2024-11-16T16:16:59,797 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:16:59,797 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:16:59,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-16T16:16:59,845 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:16:59,846 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:16:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-16T16:16:59,847 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:16:59,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:16:59,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:16:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T16:16:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T16:17:00,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-16T16:17:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:00,001 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:00,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:00,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168c1321249eda4e008c03fb59594ccf77_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773818324/Put/seqid=0 2024-11-16T16:17:00,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742433_1609 (size=12304) 2024-11-16T16:17:00,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T16:17:00,193 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/ebf73770c7bd42bf8bb6e433e79c860f as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ebf73770c7bd42bf8bb6e433e79c860f 2024-11-16T16:17:00,196 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into ebf73770c7bd42bf8bb6e433e79c860f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:00,196 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:00,196 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=12, startTime=1731773819691; duration=0sec 2024-11-16T16:17:00,196 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:00,196 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:17:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:00,426 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411168c1321249eda4e008c03fb59594ccf77_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168c1321249eda4e008c03fb59594ccf77_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/8a68749109c94d0a83593df928ea34f2, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:00,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/8a68749109c94d0a83593df928ea34f2 is 175, key is test_row_0/A:col10/1731773818324/Put/seqid=0 2024-11-16T16:17:00,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742434_1610 (size=31105) 2024-11-16T16:17:00,434 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/8a68749109c94d0a83593df928ea34f2 2024-11-16T16:17:00,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/698ecb71ace54aa0a03969e87517b9d9 is 50, key is test_row_0/B:col10/1731773818324/Put/seqid=0 2024-11-16T16:17:00,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742435_1611 (size=12151) 2024-11-16T16:17:00,467 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/698ecb71ace54aa0a03969e87517b9d9 2024-11-16T16:17:00,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/4ffa84109fb54b19b0f8ab7e13524242 is 50, key is test_row_0/C:col10/1731773818324/Put/seqid=0 2024-11-16T16:17:00,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742436_1612 (size=12151) 2024-11-16T16:17:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T16:17:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:00,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:00,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773880510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773820491, max timestamp=1731773820513Num uniques 859; 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773820491, max timestamp=1731773820513Num uniques 859; 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773820491, max timestamp=1731773820513Num uniques 859; 2024-11-16T16:17:00,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773880554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773880554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,559 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773880625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,628 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773880665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773880665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773880834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773880875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773880875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,878 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/4ffa84109fb54b19b0f8ab7e13524242 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:00,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:00,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/8a68749109c94d0a83593df928ea34f2 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2 2024-11-16T16:17:00,884 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2, entries=150, sequenceid=231, filesize=30.4 K 2024-11-16T16:17:00,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/698ecb71ace54aa0a03969e87517b9d9 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9 2024-11-16T16:17:00,888 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T16:17:00,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/4ffa84109fb54b19b0f8ab7e13524242 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242 2024-11-16T16:17:00,891 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242, entries=150, sequenceid=231, filesize=11.9 K 2024-11-16T16:17:00,892 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3818d7232c0ec9914127cf9e2f87f6dd in 892ms, sequenceid=231, compaction requested=false 2024-11-16T16:17:00,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:00,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:00,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-16T16:17:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-16T16:17:00,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-16T16:17:00,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0460 sec 2024-11-16T16:17:00,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.0500 sec 2024-11-16T16:17:00,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-16T16:17:00,995 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:00,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:17:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-16T16:17:00,998 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:17:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:00,999 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:17:00,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:17:01,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:01,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:01,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:01,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:01,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111688d4d86c740e463e85dbe818d5eaf1cd_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742437_1613 (size=12454) 2024-11-16T16:17:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773881182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773881185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773881185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773881295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:01,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773881505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:01,576 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111688d4d86c740e463e85dbe818d5eaf1cd_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111688d4d86c740e463e85dbe818d5eaf1cd_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:01,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/2fbe53d202124b0fbc5d48148b66fa9a, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:01,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/2fbe53d202124b0fbc5d48148b66fa9a is 175, key is test_row_0/A:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:01,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742438_1614 (size=31255) 2024-11-16T16:17:01,593 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=269, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/2fbe53d202124b0fbc5d48148b66fa9a 2024-11-16T16:17:01,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0d27dfa4f0614704be7077f910b1a2ed is 50, key is test_row_0/B:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:01,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742439_1615 (size=12301) 2024-11-16T16:17:01,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0d27dfa4f0614704be7077f910b1a2ed 2024-11-16T16:17:01,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/60694660d3dc492eabab714b71ad58e8 is 50, key is test_row_0/C:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:01,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742440_1616 (size=12301) 2024-11-16T16:17:01,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:01,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773881694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,696 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773881705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773881815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:01,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:01,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:01,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:02,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/60694660d3dc492eabab714b71ad58e8 2024-11-16T16:17:02,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/2fbe53d202124b0fbc5d48148b66fa9a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a 2024-11-16T16:17:02,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a, entries=150, sequenceid=269, filesize=30.5 K 2024-11-16T16:17:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/0d27dfa4f0614704be7077f910b1a2ed as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed 2024-11-16T16:17:02,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed, entries=150, sequenceid=269, filesize=12.0 K 2024-11-16T16:17:02,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/60694660d3dc492eabab714b71ad58e8 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8 2024-11-16T16:17:02,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8, entries=150, sequenceid=269, filesize=12.0 K 2024-11-16T16:17:02,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3818d7232c0ec9914127cf9e2f87f6dd in 890ms, sequenceid=269, compaction requested=true 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:17:02,038 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:17:02,038 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:02,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93909 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:17:02,041 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:02,041 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:02,041 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a621cd9056de434cbc2a84e65f9affa2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=36.2 K 2024-11-16T16:17:02,041 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=91.7 K 2024-11-16T16:17:02,041 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a] 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a621cd9056de434cbc2a84e65f9affa2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e7f0b11d2fe4a7aa8e4c1aaf0d43006, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 698ecb71ace54aa0a03969e87517b9d9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731773818311 2024-11-16T16:17:02,041 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a68749109c94d0a83593df928ea34f2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731773818311 2024-11-16T16:17:02,042 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d27dfa4f0614704be7077f910b1a2ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:02,042 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fbe53d202124b0fbc5d48148b66fa9a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:02,046 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:02,047 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:02,048 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/c0b2202233294009aaad05108d01722a is 50, key is test_row_0/B:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:02,059 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116e0f11580d6ad428eab0c39e93bc49a4c_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:02,061 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116e0f11580d6ad428eab0c39e93bc49a4c_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:02,061 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116e0f11580d6ad428eab0c39e93bc49a4c_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:02,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742441_1617 (size=12847) 2024-11-16T16:17:02,067 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/c0b2202233294009aaad05108d01722a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c0b2202233294009aaad05108d01722a 2024-11-16T16:17:02,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:02,071 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:02,073 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into c0b2202233294009aaad05108d01722a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:02,073 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:02,074 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=13, startTime=1731773822038; duration=0sec 2024-11-16T16:17:02,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:02,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:17:02,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:02,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:02,074 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:17:02,075 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:02,075 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ebf73770c7bd42bf8bb6e433e79c860f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=36.2 K 2024-11-16T16:17:02,075 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ebf73770c7bd42bf8bb6e433e79c860f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731773817144 2024-11-16T16:17:02,075 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ffa84109fb54b19b0f8ab7e13524242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731773818311 2024-11-16T16:17:02,075 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 60694660d3dc492eabab714b71ad58e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:02,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164208f145bf784945b983f086d91c43d7_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773821169/Put/seqid=0 2024-11-16T16:17:02,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742442_1618 (size=4469) 2024-11-16T16:17:02,083 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#525 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:02,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/fb45973521274726938ac9ea73dd98f4 is 175, key is test_row_0/A:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:02,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742443_1619 (size=12454) 2024-11-16T16:17:02,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:02,098 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:02,099 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/c2f66edcb0484a4a90cf18e5d5099b05 is 50, key is test_row_0/C:col10/1731773820552/Put/seqid=0 2024-11-16T16:17:02,101 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164208f145bf784945b983f086d91c43d7_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164208f145bf784945b983f086d91c43d7_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:02,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/e0e0d99e4ac74b699a573bf27dcbc9a7, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:02,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/e0e0d99e4ac74b699a573bf27dcbc9a7 is 175, key is test_row_0/A:col10/1731773821169/Put/seqid=0 2024-11-16T16:17:02,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742444_1620 (size=31801) 2024-11-16T16:17:02,111 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/fb45973521274726938ac9ea73dd98f4 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4 2024-11-16T16:17:02,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742445_1621 (size=12847) 2024-11-16T16:17:02,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742446_1622 (size=31255) 2024-11-16T16:17:02,117 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into fb45973521274726938ac9ea73dd98f4(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:02,117 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:02,117 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=13, startTime=1731773822038; duration=0sec 2024-11-16T16:17:02,117 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/c2f66edcb0484a4a90cf18e5d5099b05 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/c2f66edcb0484a4a90cf18e5d5099b05 2024-11-16T16:17:02,117 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:02,117 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:17:02,124 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into c2f66edcb0484a4a90cf18e5d5099b05(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:02,124 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:02,124 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=13, startTime=1731773822038; duration=0sec 2024-11-16T16:17:02,124 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:02,124 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:17:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:02,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773822373, max timestamp=1731773822395Num uniques 859; 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773822373, max timestamp=1731773822395Num uniques 859; 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773822373, max timestamp=1731773822395Num uniques 859; 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:02,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773882466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:02,515 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=279, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/e0e0d99e4ac74b699a573bf27dcbc9a7 2024-11-16T16:17:02,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/d90b74ce659c4079be354009f1cce704 is 50, key is test_row_0/B:col10/1731773821169/Put/seqid=0 2024-11-16T16:17:02,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742447_1623 (size=12301) 2024-11-16T16:17:02,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:02,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773882575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:02,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773882705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:02,709 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:02,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:02,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773882724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:02,727 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:02,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773882785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:02,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:02,923 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/d90b74ce659c4079be354009f1cce704 2024-11-16T16:17:02,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/6ecfdc06c12b4fd3987cb1e6ad29da25 is 50, key is test_row_0/C:col10/1731773821169/Put/seqid=0 2024-11-16T16:17:02,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742448_1624 (size=12301) 2024-11-16T16:17:03,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:03,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773883095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,098 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:03,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:03,332 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/6ecfdc06c12b4fd3987cb1e6ad29da25 2024-11-16T16:17:03,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/e0e0d99e4ac74b699a573bf27dcbc9a7 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7 2024-11-16T16:17:03,338 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7, entries=150, sequenceid=279, filesize=30.5 K 2024-11-16T16:17:03,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/d90b74ce659c4079be354009f1cce704 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704 2024-11-16T16:17:03,341 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T16:17:03,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/6ecfdc06c12b4fd3987cb1e6ad29da25 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25 2024-11-16T16:17:03,344 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25, entries=150, sequenceid=279, filesize=12.0 K 2024-11-16T16:17:03,345 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3818d7232c0ec9914127cf9e2f87f6dd in 1274ms, sequenceid=279, compaction requested=false 2024-11-16T16:17:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-16T16:17:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-16T16:17:03,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-16T16:17:03,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3470 sec 2024-11-16T16:17:03,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.3520 sec 2024-11-16T16:17:03,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:03,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-16T16:17:03,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161348b0e6f91b4bf6a7ee2a6ced02c79b_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:03,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742449_1625 (size=14994) 2024-11-16T16:17:03,623 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:03,626 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411161348b0e6f91b4bf6a7ee2a6ced02c79b_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161348b0e6f91b4bf6a7ee2a6ced02c79b_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:03,627 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b9e10777d9cb487abde4723faf6cd514, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:03,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b9e10777d9cb487abde4723faf6cd514 is 175, key is test_row_0/A:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:03,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742450_1626 (size=39949) 2024-11-16T16:17:03,641 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b9e10777d9cb487abde4723faf6cd514 2024-11-16T16:17:03,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/261ac5eb40b54b2a8309d4b5420996b5 is 50, key is test_row_0/B:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773883644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:03,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742451_1627 (size=12301) 2024-11-16T16:17:03,655 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/261ac5eb40b54b2a8309d4b5420996b5 2024-11-16T16:17:03,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/ac8e4483b6294254af407e5498a594ed is 50, key is test_row_0/C:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:03,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742452_1628 (size=12301) 2024-11-16T16:17:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773883754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:03,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:03,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773883965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:03,971 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/ac8e4483b6294254af407e5498a594ed 2024-11-16T16:17:04,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/b9e10777d9cb487abde4723faf6cd514 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514 2024-11-16T16:17:04,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514, entries=200, sequenceid=320, filesize=39.0 K 2024-11-16T16:17:04,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/261ac5eb40b54b2a8309d4b5420996b5 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5 2024-11-16T16:17:04,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T16:17:04,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/ac8e4483b6294254af407e5498a594ed as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed 2024-11-16T16:17:04,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed, entries=150, sequenceid=320, filesize=12.0 K 2024-11-16T16:17:04,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 3818d7232c0ec9914127cf9e2f87f6dd in 474ms, sequenceid=320, compaction requested=true 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:04,082 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:17:04,082 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:04,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103005 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:17:04,083 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:04,083 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:04,083 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c0b2202233294009aaad05108d01722a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=36.6 K 2024-11-16T16:17:04,083 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=100.6 K 2024-11-16T16:17:04,083 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514] 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c0b2202233294009aaad05108d01722a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb45973521274726938ac9ea73dd98f4, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting d90b74ce659c4079be354009f1cce704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731773821169 2024-11-16T16:17:04,083 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0e0d99e4ac74b699a573bf27dcbc9a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731773821169 2024-11-16T16:17:04,084 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 261ac5eb40b54b2a8309d4b5420996b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:04,084 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9e10777d9cb487abde4723faf6cd514, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:04,088 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:04,088 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:04,089 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/95d23d1981614a9d9e885ddb6061e7ba is 50, key is test_row_0/B:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:04,090 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241116223137a434ac43ff8ec62b2902ef4494_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:04,092 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241116223137a434ac43ff8ec62b2902ef4494_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:04,092 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116223137a434ac43ff8ec62b2902ef4494_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:04,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742453_1629 (size=12949) 2024-11-16T16:17:04,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742454_1630 (size=4469) 2024-11-16T16:17:04,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:04,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:04,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:04,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665ff5fcdbe674e349a324cfaf7581624_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773823632/Put/seqid=0 2024-11-16T16:17:04,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742455_1631 (size=14994) 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773824347, max timestamp=1731773824368Num uniques 859; 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773824347, max timestamp=1731773824368Num uniques 859; 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773824347, max timestamp=1731773824368Num uniques 859; 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:04,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:04,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773884469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,508 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/95d23d1981614a9d9e885ddb6061e7ba as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/95d23d1981614a9d9e885ddb6061e7ba 2024-11-16T16:17:04,514 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#534 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:04,514 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into 95d23d1981614a9d9e885ddb6061e7ba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:04,514 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:04,514 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=13, startTime=1731773824082; duration=0sec 2024-11-16T16:17:04,514 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:04,514 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:17:04,515 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-16T16:17:04,515 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/59864dd5aa424622b96df560790a3af6 is 175, key is test_row_0/A:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:04,516 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-16T16:17:04,516 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:17:04,516 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:04,516 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/c2f66edcb0484a4a90cf18e5d5099b05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=36.6 K 2024-11-16T16:17:04,517 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f66edcb0484a4a90cf18e5d5099b05, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731773820547 2024-11-16T16:17:04,517 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ecfdc06c12b4fd3987cb1e6ad29da25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731773821169 2024-11-16T16:17:04,517 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting ac8e4483b6294254af407e5498a594ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:04,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742456_1632 (size=31903) 2024-11-16T16:17:04,537 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#536 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:04,537 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/8e4dcaac33154d75ad9f47742fe473fd is 50, key is test_row_0/C:col10/1731773822459/Put/seqid=0 2024-11-16T16:17:04,540 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/59864dd5aa424622b96df560790a3af6 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6 2024-11-16T16:17:04,546 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into 59864dd5aa424622b96df560790a3af6(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:04,546 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:04,546 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=13, startTime=1731773824082; duration=0sec 2024-11-16T16:17:04,546 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:04,546 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:17:04,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742457_1633 (size=12949) 2024-11-16T16:17:04,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773884585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:04,690 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111665ff5fcdbe674e349a324cfaf7581624_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665ff5fcdbe674e349a324cfaf7581624_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:04,690 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf46b0d77e5f48c1b0532e0d52a2ac96, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:04,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf46b0d77e5f48c1b0532e0d52a2ac96 is 175, key is test_row_0/A:col10/1731773823632/Put/seqid=0 2024-11-16T16:17:04,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742458_1634 (size=39949) 2024-11-16T16:17:04,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:04,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55300 deadline: 1731773884724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:04,728 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_1' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:17:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55306 deadline: 1731773884745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:04,749 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 303fe01965e8,37355,1731773652566 for 'test_row_0' in 3818d7232c0ec9914127cf9e2f87f6dd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-16T16:17:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,750 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773884794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:04,956 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/8e4dcaac33154d75ad9f47742fe473fd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8e4dcaac33154d75ad9f47742fe473fd 2024-11-16T16:17:04,959 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into 8e4dcaac33154d75ad9f47742fe473fd(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:04,959 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:04,959 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=13, startTime=1731773824082; duration=0sec 2024-11-16T16:17:04,959 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:04,959 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:17:05,094 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf46b0d77e5f48c1b0532e0d52a2ac96 2024-11-16T16:17:05,099 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/4915d404b6444d6aa8aab1ec9a8453c0 is 50, key is test_row_0/B:col10/1731773823632/Put/seqid=0 2024-11-16T16:17:05,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:05,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773885107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:05,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:05,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742459_1635 (size=12301) 2024-11-16T16:17:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-16T16:17:05,165 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:05,165 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:17:05,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-16T16:17:05,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:17:05,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:05,166 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:17:05,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:17:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:05,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:05,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:05,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/4915d404b6444d6aa8aab1ec9a8453c0 2024-11-16T16:17:05,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/f09e2586f1994f818b90cb037d2d0802 is 50, key is test_row_0/C:col10/1731773823632/Put/seqid=0 2024-11-16T16:17:05,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742460_1636 (size=12301) 2024-11-16T16:17:05,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773885615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:05,618 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:05,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:05,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:05,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/f09e2586f1994f818b90cb037d2d0802 2024-11-16T16:17:05,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:05,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cf46b0d77e5f48c1b0532e0d52a2ac96 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96 2024-11-16T16:17:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:05,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:05,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96, entries=200, sequenceid=330, filesize=39.0 K 2024-11-16T16:17:05,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/4915d404b6444d6aa8aab1ec9a8453c0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0 2024-11-16T16:17:05,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T16:17:05,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/f09e2586f1994f818b90cb037d2d0802 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802 2024-11-16T16:17:05,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802, entries=150, sequenceid=330, filesize=12.0 K 2024-11-16T16:17:05,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 3818d7232c0ec9914127cf9e2f87f6dd in 1670ms, sequenceid=330, compaction requested=false 2024-11-16T16:17:05,948 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:06,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-16T16:17:06,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:06,082 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:06,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111640eea806a8814fefbd24f1102fb974de_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773824438/Put/seqid=0 2024-11-16T16:17:06,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742461_1637 (size=12454) 2024-11-16T16:17:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:06,508 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111640eea806a8814fefbd24f1102fb974de_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111640eea806a8814fefbd24f1102fb974de_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:06,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cd94af33d79546b2930352dd342b22ca, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cd94af33d79546b2930352dd342b22ca is 175, key is test_row_0/A:col10/1731773824438/Put/seqid=0 2024-11-16T16:17:06,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742462_1638 (size=31255) 2024-11-16T16:17:06,518 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cd94af33d79546b2930352dd342b22ca 2024-11-16T16:17:06,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/8f9552d025be41b8a731072dacde84da is 50, key is test_row_0/B:col10/1731773824438/Put/seqid=0 2024-11-16T16:17:06,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742463_1639 (size=12301) 2024-11-16T16:17:06,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:06,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:06,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:06,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773886667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:06,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:06,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773886775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:06,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:06,940 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/8f9552d025be41b8a731072dacde84da 2024-11-16T16:17:06,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/da7f645815aa49169df7a29af0addc03 is 50, key is test_row_0/C:col10/1731773824438/Put/seqid=0 2024-11-16T16:17:06,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742464_1640 (size=12301) 2024-11-16T16:17:06,964 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/da7f645815aa49169df7a29af0addc03 2024-11-16T16:17:06,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/cd94af33d79546b2930352dd342b22ca as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca 2024-11-16T16:17:06,971 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca, entries=150, sequenceid=370, filesize=30.5 K 2024-11-16T16:17:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/8f9552d025be41b8a731072dacde84da as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da 2024-11-16T16:17:06,975 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T16:17:06,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/da7f645815aa49169df7a29af0addc03 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03 2024-11-16T16:17:06,980 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03, entries=150, sequenceid=370, filesize=12.0 K 2024-11-16T16:17:06,980 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 3818d7232c0ec9914127cf9e2f87f6dd in 898ms, sequenceid=370, compaction requested=true 2024-11-16T16:17:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:06,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-16T16:17:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-16T16:17:06,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-16T16:17:06,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-16T16:17:06,987 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.8190 sec 2024-11-16T16:17:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:06,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:06,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:06,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160e72fbc942064f34a5d2831ce396a99b_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:07,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742465_1641 (size=12454) 2024-11-16T16:17:07,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:07,016 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411160e72fbc942064f34a5d2831ce396a99b_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160e72fbc942064f34a5d2831ce396a99b_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:07,019 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/76dd8cf8e3374772aab1f5855aa48c11, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:07,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/76dd8cf8e3374772aab1f5855aa48c11 is 175, key is test_row_0/A:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:07,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742466_1642 (size=31255) 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773827066, max timestamp=1731773827090Num uniques 859; 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773827066, max timestamp=1731773827090Num uniques 859; 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731773827066, max timestamp=1731773827090Num uniques 859; 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:07,093 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/303fe01965e8:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-16T16:17:07,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:07,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773887182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:07,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:07,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773887295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-16T16:17:07,315 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:07,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-16T16:17:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-16T16:17:07,317 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-16T16:17:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:07,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-16T16:17:07,318 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-16T16:17:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:07,426 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/76dd8cf8e3374772aab1f5855aa48c11 2024-11-16T16:17:07,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/a611242c1b1048aa881b0f915b0d0762 is 50, key is test_row_0/B:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:07,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742467_1643 (size=12301) 2024-11-16T16:17:07,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773887505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:07,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:07,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:07,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:07,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:07,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-16T16:17:07,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37355 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55340 deadline: 1731773887815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 2024-11-16T16:17:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 , the old value is region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,818 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-16T16:17:07,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., hostname=303fe01965e8,37355,1731773652566, seqNum=5 because the exception is null or not the one we care about 2024-11-16T16:17:07,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/a611242c1b1048aa881b0f915b0d0762 2024-11-16T16:17:07,842 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/8c4e4df4241d473fb3338c5062b1c393 is 50, key is test_row_0/C:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:07,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742468_1644 (size=12301) 2024-11-16T16:17:07,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:07,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:07,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:08,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,213 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-16T16:17:08,213 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,213 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,213 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,213 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,216 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-16T16:17:08,216 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,216 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,216 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,216 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,218 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-16T16:17:08,218 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,218 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,218 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,220 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-16T16:17:08,220 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,220 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,220 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,220 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,222 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-16T16:17:08,222 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,222 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,222 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,245 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/8c4e4df4241d473fb3338c5062b1c393 2024-11-16T16:17:08,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/76dd8cf8e3374772aab1f5855aa48c11 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11 2024-11-16T16:17:08,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11, entries=150, sequenceid=381, filesize=30.5 K 2024-11-16T16:17:08,252 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/a611242c1b1048aa881b0f915b0d0762 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762 2024-11-16T16:17:08,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T16:17:08,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/8c4e4df4241d473fb3338c5062b1c393 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393 2024-11-16T16:17:08,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393, entries=150, sequenceid=381, filesize=12.0 K 2024-11-16T16:17:08,259 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 3818d7232c0ec9914127cf9e2f87f6dd in 1271ms, sequenceid=381, compaction requested=true 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:A, priority=-2147483648, current under compaction store size is 1 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:08,259 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:B, priority=-2147483648, current under compaction store size is 2 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3818d7232c0ec9914127cf9e2f87f6dd:C, priority=-2147483648, current under compaction store size is 3 2024-11-16T16:17:08,259 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:17:08,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/A is initiating minor compaction (all files) 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/B is initiating minor compaction (all files) 2024-11-16T16:17:08,261 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/A in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,261 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/B in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,261 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=131.2 K 2024-11-16T16:17:08,261 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/95d23d1981614a9d9e885ddb6061e7ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=48.7 K 2024-11-16T16:17:08,261 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. files: [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11] 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 95d23d1981614a9d9e885ddb6061e7ba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59864dd5aa424622b96df560790a3af6, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf46b0d77e5f48c1b0532e0d52a2ac96, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731773823620 2024-11-16T16:17:08,261 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 4915d404b6444d6aa8aab1ec9a8453c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731773823632 2024-11-16T16:17:08,262 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd94af33d79546b2930352dd342b22ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773824438 2024-11-16T16:17:08,262 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f9552d025be41b8a731072dacde84da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773824438 2024-11-16T16:17:08,262 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76dd8cf8e3374772aab1f5855aa48c11, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731773826642 2024-11-16T16:17:08,262 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting a611242c1b1048aa881b0f915b0d0762, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731773826642 2024-11-16T16:17:08,269 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:08,270 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#B#compaction#545 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:08,270 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411164c7affc4b7d349a09fe6f7d584c38b4b_3818d7232c0ec9914127cf9e2f87f6dd store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:08,270 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/5f061fe935c34465b52c26f3b2c30969 is 50, key is test_row_0/B:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:08,274 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411164c7affc4b7d349a09fe6f7d584c38b4b_3818d7232c0ec9914127cf9e2f87f6dd, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:08,274 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411164c7affc4b7d349a09fe6f7d584c38b4b_3818d7232c0ec9914127cf9e2f87f6dd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:08,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742469_1645 (size=13085) 2024-11-16T16:17:08,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742470_1646 (size=4469) 2024-11-16T16:17:08,286 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#A#compaction#546 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:08,287 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/7487ad978d214d3aa0b5f097c83c790a is 175, key is test_row_0/A:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:08,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742471_1647 (size=32039) 2024-11-16T16:17:08,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37355 {}] regionserver.HRegion(8855): Flush requested on 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:08,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-16T16:17:08,326 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-16T16:17:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:08,326 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,326 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:08,327 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-16T16:17:08,327 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee50403c8bf441f88785eb47e8137fff_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773827174/Put/seqid=0 2024-11-16T16:17:08,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742472_1648 (size=12454) 2024-11-16T16:17:08,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:08,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,697 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/5f061fe935c34465b52c26f3b2c30969 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5f061fe935c34465b52c26f3b2c30969 2024-11-16T16:17:08,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,699 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/7487ad978d214d3aa0b5f097c83c790a as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/7487ad978d214d3aa0b5f097c83c790a 2024-11-16T16:17:08,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,702 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/B of 3818d7232c0ec9914127cf9e2f87f6dd into 5f061fe935c34465b52c26f3b2c30969(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:08,702 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:08,702 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/B, priority=12, startTime=1731773828259; duration=0sec 2024-11-16T16:17:08,702 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-16T16:17:08,702 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:B 2024-11-16T16:17:08,702 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-16T16:17:08,703 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/A of 3818d7232c0ec9914127cf9e2f87f6dd into 7487ad978d214d3aa0b5f097c83c790a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:08,703 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:08,703 INFO [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/A, priority=12, startTime=1731773828259; duration=0sec 2024-11-16T16:17:08,703 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:08,703 DEBUG [RS:0;303fe01965e8:37355-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:A 2024-11-16T16:17:08,703 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-16T16:17:08,703 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1541): 3818d7232c0ec9914127cf9e2f87f6dd/C is initiating minor compaction (all files) 2024-11-16T16:17:08,703 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3818d7232c0ec9914127cf9e2f87f6dd/C in TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,704 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8e4dcaac33154d75ad9f47742fe473fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393] into tmpdir=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp, totalSize=48.7 K 2024-11-16T16:17:08,704 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4dcaac33154d75ad9f47742fe473fd, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731773822431 2024-11-16T16:17:08,704 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting f09e2586f1994f818b90cb037d2d0802, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731773823632 2024-11-16T16:17:08,704 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting da7f645815aa49169df7a29af0addc03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731773824438 2024-11-16T16:17:08,705 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c4e4df4241d473fb3338c5062b1c393, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731773826642 2024-11-16T16:17:08,712 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3818d7232c0ec9914127cf9e2f87f6dd#C#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-16T16:17:08,713 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/4ffd2efe91824e4e9a48a5f595ec5bdd is 50, key is test_row_0/C:col10/1731773826987/Put/seqid=0 2024-11-16T16:17:08,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742473_1649 (size=13085) 2024-11-16T16:17:08,735 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:08,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116ee50403c8bf441f88785eb47e8137fff_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee50403c8bf441f88785eb47e8137fff_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:08,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9e8c7c3c73c14461b4f9ed7aec0714d0, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:08,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9e8c7c3c73c14461b4f9ed7aec0714d0 is 175, key is test_row_0/A:col10/1731773827174/Put/seqid=0 2024-11-16T16:17:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742474_1650 (size=31255) 2024-11-16T16:17:08,746 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-16T16:17:08,746 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,746 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,746 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,767 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-11-16T16:17:08,767 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,767 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,767 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,767 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:08,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:08,879 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-16T16:17:08,879 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,879 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,879 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-16T16:17:08,879 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-16T16:17:08,879 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,879 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:08,880 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:08,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:09,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,121 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/4ffd2efe91824e4e9a48a5f595ec5bdd as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffd2efe91824e4e9a48a5f595ec5bdd 2024-11-16T16:17:09,126 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 3818d7232c0ec9914127cf9e2f87f6dd/C of 3818d7232c0ec9914127cf9e2f87f6dd into 4ffd2efe91824e4e9a48a5f595ec5bdd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-16T16:17:09,126 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:09,126 INFO [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd., storeName=3818d7232c0ec9914127cf9e2f87f6dd/C, priority=12, startTime=1731773828259; duration=0sec 2024-11-16T16:17:09,126 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-16T16:17:09,126 DEBUG [RS:0;303fe01965e8:37355-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3818d7232c0ec9914127cf9e2f87f6dd:C 2024-11-16T16:17:09,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9e8c7c3c73c14461b4f9ed7aec0714d0 2024-11-16T16:17:09,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/bc1937a04f5942b3ac09cfd9c389570d is 50, key is test_row_0/B:col10/1731773827174/Put/seqid=0 2024-11-16T16:17:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742475_1651 (size=12301) 2024-11-16T16:17:09,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:09,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/bc1937a04f5942b3ac09cfd9c389570d 2024-11-16T16:17:09,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/d8716674c8b0487d91e25a66d6b6f632 is 50, key is test_row_0/C:col10/1731773827174/Put/seqid=0 2024-11-16T16:17:09,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742476_1652 (size=12301) 2024-11-16T16:17:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. as already flushing 2024-11-16T16:17:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:09,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T16:17:09,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/d8716674c8b0487d91e25a66d6b6f632 2024-11-16T16:17:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9e8c7c3c73c14461b4f9ed7aec0714d0 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9e8c7c3c73c14461b4f9ed7aec0714d0 2024-11-16T16:17:09,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9e8c7c3c73c14461b4f9ed7aec0714d0, entries=150, sequenceid=418, filesize=30.5 K 2024-11-16T16:17:09,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/bc1937a04f5942b3ac09cfd9c389570d as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/bc1937a04f5942b3ac09cfd9c389570d 2024-11-16T16:17:09,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/bc1937a04f5942b3ac09cfd9c389570d, entries=150, sequenceid=418, filesize=12.0 K 2024-11-16T16:17:09,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/d8716674c8b0487d91e25a66d6b6f632 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/d8716674c8b0487d91e25a66d6b6f632 2024-11-16T16:17:09,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/d8716674c8b0487d91e25a66d6b6f632, entries=150, sequenceid=418, filesize=12.0 K 2024-11-16T16:17:09,980 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=26.84 KB/27480 for 3818d7232c0ec9914127cf9e2f87f6dd in 1654ms, sequenceid=418, compaction requested=false 2024-11-16T16:17:09,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:10,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37355 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-16T16:17:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:10,089 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 3818d7232c0ec9914127cf9e2f87f6dd 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-16T16:17:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=A 2024-11-16T16:17:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=B 2024-11-16T16:17:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 3818d7232c0ec9914127cf9e2f87f6dd, store=C 2024-11-16T16:17:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-16T16:17:10,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7756f55247640dea06b696e57743c9f_3818d7232c0ec9914127cf9e2f87f6dd is 50, key is test_row_0/A:col10/1731773828765/Put/seqid=0 2024-11-16T16:17:10,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742477_1653 (size=12454) 2024-11-16T16:17:10,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T16:17:10,511 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241116f7756f55247640dea06b696e57743c9f_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7756f55247640dea06b696e57743c9f_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:10,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9ff1a486518c49ceb7675d2d45586a60, store: [table=TestAcidGuarantees family=A region=3818d7232c0ec9914127cf9e2f87f6dd] 2024-11-16T16:17:10,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9ff1a486518c49ceb7675d2d45586a60 is 175, key is test_row_0/A:col10/1731773828765/Put/seqid=0 2024-11-16T16:17:10,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742478_1654 (size=31255) 2024-11-16T16:17:10,853 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-16T16:17:10,921 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=428, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9ff1a486518c49ceb7675d2d45586a60 2024-11-16T16:17:10,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/c71ec5fa39ad46d482f0e273fb639727 is 50, key is test_row_0/B:col10/1731773828765/Put/seqid=0 2024-11-16T16:17:10,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742479_1655 (size=12301) 2024-11-16T16:17:11,332 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/c71ec5fa39ad46d482f0e273fb639727 2024-11-16T16:17:11,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/b3c71f52d5374878a5e3fc3f48217120 is 50, key is test_row_0/C:col10/1731773828765/Put/seqid=0 2024-11-16T16:17:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742480_1656 (size=12301) 2024-11-16T16:17:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:11,744 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/b3c71f52d5374878a5e3fc3f48217120 2024-11-16T16:17:11,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/A/9ff1a486518c49ceb7675d2d45586a60 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9ff1a486518c49ceb7675d2d45586a60 2024-11-16T16:17:11,755 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9ff1a486518c49ceb7675d2d45586a60, entries=150, sequenceid=428, filesize=30.5 K 2024-11-16T16:17:11,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/B/c71ec5fa39ad46d482f0e273fb639727 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c71ec5fa39ad46d482f0e273fb639727 2024-11-16T16:17:11,758 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c71ec5fa39ad46d482f0e273fb639727, entries=150, sequenceid=428, filesize=12.0 K 2024-11-16T16:17:11,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/.tmp/C/b3c71f52d5374878a5e3fc3f48217120 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/b3c71f52d5374878a5e3fc3f48217120 2024-11-16T16:17:11,761 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/b3c71f52d5374878a5e3fc3f48217120, entries=150, sequenceid=428, filesize=12.0 K 2024-11-16T16:17:11,762 INFO [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 3818d7232c0ec9914127cf9e2f87f6dd in 1673ms, sequenceid=428, compaction requested=true 2024-11-16T16:17:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 3818d7232c0ec9914127cf9e2f87f6dd: 2024-11-16T16:17:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:11,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/303fe01965e8:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-16T16:17:11,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-16T16:17:11,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-16T16:17:11,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4450 sec 2024-11-16T16:17:11,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 4.4490 sec 2024-11-16T16:17:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-16T16:17:15,516 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2794 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2783 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2775 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2795 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2788 2024-11-16T16:17:15,517 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-16T16:17:15,518 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:17:15,518 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:17:15,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:15,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:15,519 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:15,520 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-16T16:17:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T16:17:15,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773835525"}]},"ts":"1731773835525"} 2024-11-16T16:17:15,526 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-16T16:17:15,526 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-16T16:17:15,527 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-16T16:17:15,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, UNASSIGN}] 2024-11-16T16:17:15,530 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, UNASSIGN 2024-11-16T16:17:15,530 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=CLOSING, regionLocation=303fe01965e8,37355,1731773652566 2024-11-16T16:17:15,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=172, ppid=171, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, UNASSIGN because future has completed 2024-11-16T16:17:15,532 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-16T16:17:15,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566}] 2024-11-16T16:17:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T16:17:15,685 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(122): Close 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,685 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-16T16:17:15,685 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1722): Closing 3818d7232c0ec9914127cf9e2f87f6dd, disabling compactions & flushes 2024-11-16T16:17:15,686 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:15,686 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:15,686 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. after waiting 0 ms 2024-11-16T16:17:15,686 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:15,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11] to archive 2024-11-16T16:17:15,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:17:15,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9dfb455ae012466b97c9266c51e8e330 2024-11-16T16:17:15,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf2afe4f45004a4db6630161553cbd05 2024-11-16T16:17:15,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4f9c4465c0574f2cad72de575ab3f739 2024-11-16T16:17:15,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/4feb80f2cdb142b5aa8aea8728826f79 2024-11-16T16:17:15,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/82e2fd870cc148328d679dd86e4af6ae 2024-11-16T16:17:15,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/add94e286e514f29a2cf45c61526d754 2024-11-16T16:17:15,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/41c4dd93b9da4d9a983a5e2e7a3fe5fc 2024-11-16T16:17:15,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b0b1a48dd01246658b52b3a5cd84082f 2024-11-16T16:17:15,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/6942ef0f038d49b6b9b5ff58fdb5b2c8 2024-11-16T16:17:15,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/f536d41e1a884559a7a978b434a54a5d 2024-11-16T16:17:15,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/3c40ac7dba80441f80b191fe9ffabbad 2024-11-16T16:17:15,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/c8b7310250d54bebac723950db69d3ba 2024-11-16T16:17:15,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/1e7f0b11d2fe4a7aa8e4c1aaf0d43006 2024-11-16T16:17:15,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/44eac1f2bcac4554a58d81096468b314 2024-11-16T16:17:15,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/8a68749109c94d0a83593df928ea34f2 2024-11-16T16:17:15,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/fb45973521274726938ac9ea73dd98f4 2024-11-16T16:17:15,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/2fbe53d202124b0fbc5d48148b66fa9a 2024-11-16T16:17:15,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/e0e0d99e4ac74b699a573bf27dcbc9a7 2024-11-16T16:17:15,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/b9e10777d9cb487abde4723faf6cd514 2024-11-16T16:17:15,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/59864dd5aa424622b96df560790a3af6 2024-11-16T16:17:15,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cf46b0d77e5f48c1b0532e0d52a2ac96 2024-11-16T16:17:15,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/cd94af33d79546b2930352dd342b22ca 2024-11-16T16:17:15,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/76dd8cf8e3374772aab1f5855aa48c11 2024-11-16T16:17:15,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/70ba82ea7b924bc2819464116b10ef95, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0b393cd9413a4dbb9ec18dd2b0b46dd6, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/ee1559fbcd154080856b1e623528b7b8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a621cd9056de434cbc2a84e65f9affa2, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c0b2202233294009aaad05108d01722a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/95d23d1981614a9d9e885ddb6061e7ba, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762] to archive 2024-11-16T16:17:15,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:17:15,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/3baee1edf02740da8efb4ca44f988cd7 2024-11-16T16:17:15,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/38924e94a3944f5db89794ecbff709e0 2024-11-16T16:17:15,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/70ba82ea7b924bc2819464116b10ef95 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/70ba82ea7b924bc2819464116b10ef95 2024-11-16T16:17:15,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/9c97417d8c014d4d9d524299811ae39e 2024-11-16T16:17:15,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0aae01e9901d4e1b8bf19373358b3af3 2024-11-16T16:17:15,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0b393cd9413a4dbb9ec18dd2b0b46dd6 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0b393cd9413a4dbb9ec18dd2b0b46dd6 2024-11-16T16:17:15,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5e743b9221eb4336a73b09b7197d065d 2024-11-16T16:17:15,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/f786a4998d4b40e9ad903f15d102674e 2024-11-16T16:17:15,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/ee1559fbcd154080856b1e623528b7b8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/ee1559fbcd154080856b1e623528b7b8 2024-11-16T16:17:15,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/cb23ad01fe9d48148a750c80ee34e2b9 2024-11-16T16:17:15,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b1faffc3ca774129a3cc4c26ed2dd390 2024-11-16T16:17:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/b0244856f7854f58b6f310c7c537de85 2024-11-16T16:17:15,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a621cd9056de434cbc2a84e65f9affa2 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a621cd9056de434cbc2a84e65f9affa2 2024-11-16T16:17:15,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/7e5965232f46415ea8ffdb974bfb4476 2024-11-16T16:17:15,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/698ecb71ace54aa0a03969e87517b9d9 2024-11-16T16:17:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c0b2202233294009aaad05108d01722a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c0b2202233294009aaad05108d01722a 2024-11-16T16:17:15,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/0d27dfa4f0614704be7077f910b1a2ed 2024-11-16T16:17:15,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/d90b74ce659c4079be354009f1cce704 2024-11-16T16:17:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/95d23d1981614a9d9e885ddb6061e7ba to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/95d23d1981614a9d9e885ddb6061e7ba 2024-11-16T16:17:15,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/261ac5eb40b54b2a8309d4b5420996b5 2024-11-16T16:17:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/4915d404b6444d6aa8aab1ec9a8453c0 2024-11-16T16:17:15,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/8f9552d025be41b8a731072dacde84da 2024-11-16T16:17:15,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/a611242c1b1048aa881b0f915b0d0762 2024-11-16T16:17:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6f439ca5f0814a8db0c7f7a543669782, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/a12af6c370084047a48c60efc0f46769, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/7df355a9a8eb4a2aa700d232947aa42a, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ebf73770c7bd42bf8bb6e433e79c860f, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/c2f66edcb0484a4a90cf18e5d5099b05, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8e4dcaac33154d75ad9f47742fe473fd, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393] to archive 2024-11-16T16:17:15,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-16T16:17:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/805f99d4c8a54d479d73845fa3c70eb5 2024-11-16T16:17:15,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e1b97a96c7624a0b90aeff13f5de4082 2024-11-16T16:17:15,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6f439ca5f0814a8db0c7f7a543669782 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6f439ca5f0814a8db0c7f7a543669782 2024-11-16T16:17:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/66e5bf1e6ab848078ebef8224c8f0fa5 2024-11-16T16:17:15,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/74c878559b6a4f2c80a2551a3f7a9a27 2024-11-16T16:17:15,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/a12af6c370084047a48c60efc0f46769 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/a12af6c370084047a48c60efc0f46769 2024-11-16T16:17:15,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/20b25686069049ffa2ed6ed236fbca27 2024-11-16T16:17:15,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/e3887d953aed4a2f8b0d045a6b532b5a 2024-11-16T16:17:15,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/7df355a9a8eb4a2aa700d232947aa42a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/7df355a9a8eb4a2aa700d232947aa42a 2024-11-16T16:17:15,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/617e9f5b32354bef8281a6e9b4252437 2024-11-16T16:17:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/930ef3868a574774ba1a259dfa458e83 2024-11-16T16:17:15,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/06a3cbc73a20487d80973937a9c0ee58 2024-11-16T16:17:15,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ebf73770c7bd42bf8bb6e433e79c860f to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ebf73770c7bd42bf8bb6e433e79c860f 2024-11-16T16:17:15,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/bcad65d051a9479bb43582b69fdff365 2024-11-16T16:17:15,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffa84109fb54b19b0f8ab7e13524242 2024-11-16T16:17:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/c2f66edcb0484a4a90cf18e5d5099b05 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/c2f66edcb0484a4a90cf18e5d5099b05 2024-11-16T16:17:15,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/60694660d3dc492eabab714b71ad58e8 2024-11-16T16:17:15,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/6ecfdc06c12b4fd3987cb1e6ad29da25 2024-11-16T16:17:15,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8e4dcaac33154d75ad9f47742fe473fd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8e4dcaac33154d75ad9f47742fe473fd 2024-11-16T16:17:15,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/ac8e4483b6294254af407e5498a594ed 2024-11-16T16:17:15,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/f09e2586f1994f818b90cb037d2d0802 2024-11-16T16:17:15,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/da7f645815aa49169df7a29af0addc03 2024-11-16T16:17:15,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/8c4e4df4241d473fb3338c5062b1c393 2024-11-16T16:17:15,756 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits/431.seqid, newMaxSeqId=431, maxSeqId=4 2024-11-16T16:17:15,757 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd. 2024-11-16T16:17:15,757 DEBUG [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] regionserver.HRegion(1676): Region close journal for 3818d7232c0ec9914127cf9e2f87f6dd: Waiting for close lock at 1731773835685Running coprocessor pre-close hooks at 1731773835685Disabling compacts and flushes for region at 1731773835685Disabling writes for close at 1731773835686 (+1 ms)Writing region close event to WAL at 1731773835753 (+67 ms)Running coprocessor post-close hooks at 1731773835757 (+4 ms)Closed at 1731773835757 2024-11-16T16:17:15,758 INFO [RS_CLOSE_REGION-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_REGION, pid=173}] handler.UnassignRegionHandler(157): Closed 3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,759 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=172 updating hbase:meta row=3818d7232c0ec9914127cf9e2f87f6dd, regionState=CLOSED 2024-11-16T16:17:15,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=173, ppid=172, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 because future has completed 2024-11-16T16:17:15,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-16T16:17:15,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseRegionProcedure 3818d7232c0ec9914127cf9e2f87f6dd, server=303fe01965e8,37355,1731773652566 in 230 msec 2024-11-16T16:17:15,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-11-16T16:17:15,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3818d7232c0ec9914127cf9e2f87f6dd, UNASSIGN in 235 msec 2024-11-16T16:17:15,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-16T16:17:15,768 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 239 msec 2024-11-16T16:17:15,769 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731773835769"}]},"ts":"1731773835769"} 2024-11-16T16:17:15,771 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-16T16:17:15,771 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-16T16:17:15,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 251 msec 2024-11-16T16:17:15,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-16T16:17:15,847 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:15,848 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-16T16:17:15,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,852 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=174, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T16:17:15,854 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=174, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,857 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,862 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C, FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits] 2024-11-16T16:17:15,865 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/7487ad978d214d3aa0b5f097c83c790a to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/7487ad978d214d3aa0b5f097c83c790a 2024-11-16T16:17:15,866 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9e8c7c3c73c14461b4f9ed7aec0714d0 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9e8c7c3c73c14461b4f9ed7aec0714d0 2024-11-16T16:17:15,867 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9ff1a486518c49ceb7675d2d45586a60 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/A/9ff1a486518c49ceb7675d2d45586a60 2024-11-16T16:17:15,869 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5f061fe935c34465b52c26f3b2c30969 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/5f061fe935c34465b52c26f3b2c30969 2024-11-16T16:17:15,870 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/bc1937a04f5942b3ac09cfd9c389570d to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/bc1937a04f5942b3ac09cfd9c389570d 2024-11-16T16:17:15,870 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c71ec5fa39ad46d482f0e273fb639727 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/B/c71ec5fa39ad46d482f0e273fb639727 2024-11-16T16:17:15,872 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffd2efe91824e4e9a48a5f595ec5bdd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/4ffd2efe91824e4e9a48a5f595ec5bdd 2024-11-16T16:17:15,872 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/b3c71f52d5374878a5e3fc3f48217120 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/b3c71f52d5374878a5e3fc3f48217120 2024-11-16T16:17:15,873 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/d8716674c8b0487d91e25a66d6b6f632 to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/C/d8716674c8b0487d91e25a66d6b6f632 2024-11-16T16:17:15,874 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits/431.seqid to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd/recovered.edits/431.seqid 2024-11-16T16:17:15,875 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/default/TestAcidGuarantees/3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,875 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-16T16:17:15,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:17:15,876 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-16T16:17:15,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609b07bba518545248c1d583991b9c198_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111609b07bba518545248c1d583991b9c198_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a15cd87bc5a458b9de80ed2ff2913a5_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160a15cd87bc5a458b9de80ed2ff2913a5_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,879 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160e72fbc942064f34a5d2831ce396a99b_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411160e72fbc942064f34a5d2831ce396a99b_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161348b0e6f91b4bf6a7ee2a6ced02c79b_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411161348b0e6f91b4bf6a7ee2a6ced02c79b_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111640eea806a8814fefbd24f1102fb974de_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111640eea806a8814fefbd24f1102fb974de_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,882 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164208f145bf784945b983f086d91c43d7_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411164208f145bf784945b983f086d91c43d7_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665ff5fcdbe674e349a324cfaf7581624_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111665ff5fcdbe674e349a324cfaf7581624_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116716584e887f74dcb92367ea4e940a47c_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116716584e887f74dcb92367ea4e940a47c_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cb70eebe6274952a2bf2ee5ff3459ff_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411167cb70eebe6274952a2bf2ee5ff3459ff_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111688d4d86c740e463e85dbe818d5eaf1cd_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111688d4d86c740e463e85dbe818d5eaf1cd_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,887 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168c1321249eda4e008c03fb59594ccf77_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168c1321249eda4e008c03fb59594ccf77_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168cb3943dcc0847e7b2be5b842c630ab9_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411168cb3943dcc0847e7b2be5b842c630ab9_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111695f2e29762784a9db8faa2898d78b985_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111695f2e29762784a9db8faa2898d78b985_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1cbc85993d549148676dbc1fb0b5cb9_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116c1cbc85993d549148676dbc1fb0b5cb9_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d34e47b55e434e1598887e1238a4ca93_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116d34e47b55e434e1598887e1238a4ca93_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,891 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dd887cc27639457198d9ebf5f0caf312_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116dd887cc27639457198d9ebf5f0caf312_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee50403c8bf441f88785eb47e8137fff_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116ee50403c8bf441f88785eb47e8137fff_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,892 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7756f55247640dea06b696e57743c9f_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116f7756f55247640dea06b696e57743c9f_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,893 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcaa06fb8317484a97c40b79b667fc50_3818d7232c0ec9914127cf9e2f87f6dd to hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241116fcaa06fb8317484a97c40b79b667fc50_3818d7232c0ec9914127cf9e2f87f6dd 2024-11-16T16:17:15,894 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-16T16:17:15,895 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=174, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,897 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-16T16:17:15,899 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-16T16:17:15,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=174, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-16T16:17:15,900 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731773835900"}]},"ts":"9223372036854775807"} 2024-11-16T16:17:15,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-16T16:17:15,902 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3818d7232c0ec9914127cf9e2f87f6dd, NAME => 'TestAcidGuarantees,,1731773805080.3818d7232c0ec9914127cf9e2f87f6dd.', STARTKEY => '', ENDKEY => ''}] 2024-11-16T16:17:15,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-16T16:17:15,902 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731773835902"}]},"ts":"9223372036854775807"} 2024-11-16T16:17:15,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-16T16:17:15,904 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=174, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-16T16:17:15,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 56 msec 2024-11-16T16:17:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45907 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-16T16:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-16T16:17:15,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-16T16:17:15,975 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246 (was 248), OpenFileDescriptor=453 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=426 (was 394) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4363 (was 3830) - AvailableMemoryMB LEAK? - 2024-11-16T16:17:15,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T16:17:15,975 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:17:15,975 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:17:15,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:15,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:15,976 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T16:17:15,976 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T16:17:15,976 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1372924334, stopped=false 2024-11-16T16:17:15,976 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=303fe01965e8,45907,1731773651719 2024-11-16T16:17:16,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T16:17:16,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T16:17:16,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:17:16,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:17:16,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T16:17:16,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T16:17:16,030 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T16:17:16,030 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T16:17:16,030 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:17:16,030 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:16,031 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '303fe01965e8,37355,1731773652566' ***** 2024-11-16T16:17:16,031 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T16:17:16,031 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T16:17:16,031 INFO [RS:0;303fe01965e8:37355 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T16:17:16,031 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T16:17:16,031 INFO [RS:0;303fe01965e8:37355 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(959): stopping server 303fe01965e8,37355,1731773652566 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;303fe01965e8:37355. 2024-11-16T16:17:16,032 DEBUG [RS:0;303fe01965e8:37355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T16:17:16,032 DEBUG [RS:0;303fe01965e8:37355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T16:17:16,032 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T16:17:16,033 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T16:17:16,033 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T16:17:16,033 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:16,033 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T16:17:16,033 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T16:17:16,033 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T16:17:16,033 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T16:17:16,033 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T16:17:16,033 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-16T16:17:16,035 INFO [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-16T16:17:16,035 INFO [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-16T16:17:16,035 INFO [regionserver/303fe01965e8:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-16T16:17:16,040 INFO [regionserver/303fe01965e8:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T16:17:16,053 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/info/f0406a768b7b49569070827a836f59ce is 95, key is TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22./info:/1731773684591/DeleteFamily/seqid=0 2024-11-16T16:17:16,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742481_1657 (size=6514) 2024-11-16T16:17:16,233 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:16,433 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:16,457 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/info/f0406a768b7b49569070827a836f59ce 2024-11-16T16:17:16,475 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/ns/664fc653e40347d2972843a4080e4807 is 93, key is TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22./ns:/1731773684547/DeleteFamily/seqid=0 2024-11-16T16:17:16,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742482_1658 (size=5906) 2024-11-16T16:17:16,634 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:16,834 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:16,880 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/ns/664fc653e40347d2972843a4080e4807 2024-11-16T16:17:16,904 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/rep_barrier/e7efa6efbca647ff91ae2fe8adacc595 is 102, key is TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22./rep_barrier:/1731773684547/DeleteFamily/seqid=0 2024-11-16T16:17:16,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742483_1659 (size=6025) 2024-11-16T16:17:17,034 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:17,234 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:17,309 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/rep_barrier/e7efa6efbca647ff91ae2fe8adacc595 2024-11-16T16:17:17,335 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/table/9a3608a3e84b4c5ba62f16700d85396c is 96, key is TestAcidGuarantees,,1731773657658.8161ddf0266b9f8b7a8b291995378f22./table:/1731773684547/DeleteFamily/seqid=0 2024-11-16T16:17:17,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742484_1660 (size=5936) 2024-11-16T16:17:17,435 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:17,635 DEBUG [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T16:17:17,739 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/table/9a3608a3e84b4c5ba62f16700d85396c 2024-11-16T16:17:17,749 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/info/f0406a768b7b49569070827a836f59ce as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/info/f0406a768b7b49569070827a836f59ce 2024-11-16T16:17:17,754 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/info/f0406a768b7b49569070827a836f59ce, entries=12, sequenceid=90, filesize=6.4 K 2024-11-16T16:17:17,755 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/ns/664fc653e40347d2972843a4080e4807 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/ns/664fc653e40347d2972843a4080e4807 2024-11-16T16:17:17,760 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/ns/664fc653e40347d2972843a4080e4807, entries=8, sequenceid=90, filesize=5.8 K 2024-11-16T16:17:17,761 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/rep_barrier/e7efa6efbca647ff91ae2fe8adacc595 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/rep_barrier/e7efa6efbca647ff91ae2fe8adacc595 2024-11-16T16:17:17,763 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/rep_barrier/e7efa6efbca647ff91ae2fe8adacc595, entries=6, sequenceid=90, filesize=5.9 K 2024-11-16T16:17:17,764 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/.tmp/table/9a3608a3e84b4c5ba62f16700d85396c as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/table/9a3608a3e84b4c5ba62f16700d85396c 2024-11-16T16:17:17,767 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/table/9a3608a3e84b4c5ba62f16700d85396c, entries=7, sequenceid=90, filesize=5.8 K 2024-11-16T16:17:17,768 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1735ms, sequenceid=90, compaction requested=false 2024-11-16T16:17:17,776 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-16T16:17:17,777 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T16:17:17,777 INFO [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T16:17:17,777 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731773836033Running coprocessor pre-close hooks at 1731773836033Disabling compacts and flushes for region at 1731773836033Disabling writes for close at 1731773836033Obtaining lock to block concurrent updates at 1731773836033Preparing flush snapshotting stores in 1588230740 at 1731773836033Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731773836034 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731773836034Flushing 1588230740/info: creating writer at 1731773836034Flushing 1588230740/info: appending metadata at 1731773836053 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731773836053Flushing 1588230740/ns: creating writer at 1731773836462 (+409 ms)Flushing 1588230740/ns: appending metadata at 1731773836475 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731773836475Flushing 1588230740/rep_barrier: creating writer at 1731773836887 (+412 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731773836904 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731773836904Flushing 1588230740/table: creating writer at 1731773837317 (+413 ms)Flushing 1588230740/table: appending metadata at 1731773837335 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731773837335Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26f7841a: reopening flushed file at 1731773837747 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c356dc0: reopening flushed file at 1731773837754 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ab4037: reopening flushed file at 1731773837760 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf5e453: reopening flushed file at 1731773837764 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1735ms, sequenceid=90, compaction requested=false at 1731773837768 (+4 ms)Writing region close event to WAL at 1731773837773 (+5 ms)Running coprocessor post-close hooks at 1731773837776 (+3 ms)Closed at 1731773837777 (+1 ms) 2024-11-16T16:17:17,777 DEBUG [RS_CLOSE_META-regionserver/303fe01965e8:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T16:17:17,835 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(976): stopping server 303fe01965e8,37355,1731773652566; all regions closed. 2024-11-16T16:17:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741834_1010 (size=25344) 2024-11-16T16:17:17,841 DEBUG [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/oldWALs 2024-11-16T16:17:17,842 INFO [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 303fe01965e8%2C37355%2C1731773652566.meta:.meta(num 1731773657161) 2024-11-16T16:17:17,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741832_1008 (size=17909182) 2024-11-16T16:17:17,845 DEBUG [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/oldWALs 2024-11-16T16:17:17,845 INFO [RS:0;303fe01965e8:37355 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 303fe01965e8%2C37355%2C1731773652566:(num 1731773656201) 2024-11-16T16:17:17,845 DEBUG [RS:0;303fe01965e8:37355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T16:17:17,845 INFO [RS:0;303fe01965e8:37355 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T16:17:17,845 INFO [RS:0;303fe01965e8:37355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T16:17:17,846 INFO [RS:0;303fe01965e8:37355 {}] hbase.ChoreService(370): Chore service for: regionserver/303fe01965e8:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T16:17:17,846 INFO [RS:0;303fe01965e8:37355 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T16:17:17,846 INFO [regionserver/303fe01965e8:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T16:17:17,846 INFO [RS:0;303fe01965e8:37355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37355 2024-11-16T16:17:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T16:17:17,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/303fe01965e8,37355,1731773652566 2024-11-16T16:17:17,956 INFO [RS:0;303fe01965e8:37355 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T16:17:17,967 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [303fe01965e8,37355,1731773652566] 2024-11-16T16:17:17,977 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/303fe01965e8,37355,1731773652566 already deleted, retry=false 2024-11-16T16:17:17,977 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 303fe01965e8,37355,1731773652566 expired; onlineServers=0 2024-11-16T16:17:17,978 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '303fe01965e8,45907,1731773651719' ***** 2024-11-16T16:17:17,978 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T16:17:17,978 INFO [M:0;303fe01965e8:45907 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T16:17:17,978 INFO [M:0;303fe01965e8:45907 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T16:17:17,978 DEBUG [M:0;303fe01965e8:45907 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T16:17:17,979 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T16:17:17,979 DEBUG [M:0;303fe01965e8:45907 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T16:17:17,979 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731773655868 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.large.0-1731773655868,5,FailOnTimeoutGroup] 2024-11-16T16:17:17,979 DEBUG [master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731773655872 {}] cleaner.HFileCleaner(306): Exit Thread[master/303fe01965e8:0:becomeActiveMaster-HFileCleaner.small.0-1731773655872,5,FailOnTimeoutGroup] 2024-11-16T16:17:17,980 INFO [M:0;303fe01965e8:45907 {}] hbase.ChoreService(370): Chore service for: master/303fe01965e8:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T16:17:17,980 INFO [M:0;303fe01965e8:45907 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T16:17:17,980 DEBUG [M:0;303fe01965e8:45907 {}] master.HMaster(1795): Stopping service threads 2024-11-16T16:17:17,980 INFO [M:0;303fe01965e8:45907 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T16:17:17,981 INFO [M:0;303fe01965e8:45907 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T16:17:17,981 ERROR [M:0;303fe01965e8:45907 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-16T16:17:17,983 INFO [M:0;303fe01965e8:45907 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T16:17:17,983 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T16:17:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T16:17:17,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T16:17:17,987 DEBUG [M:0;303fe01965e8:45907 {}] zookeeper.ZKUtil(347): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T16:17:17,987 WARN [M:0;303fe01965e8:45907 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T16:17:17,988 INFO [M:0;303fe01965e8:45907 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/.lastflushedseqids 2024-11-16T16:17:17,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742485_1661 (size=72) 2024-11-16T16:17:18,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T16:17:18,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37355-0x101446cd3870001, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T16:17:18,068 INFO [RS:0;303fe01965e8:37355 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T16:17:18,069 INFO [RS:0;303fe01965e8:37355 {}] regionserver.HRegionServer(1031): Exiting; stopping=303fe01965e8,37355,1731773652566; zookeeper connection closed. 2024-11-16T16:17:18,069 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2afb095b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2afb095b 2024-11-16T16:17:18,070 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-16T16:17:18,398 INFO [M:0;303fe01965e8:45907 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T16:17:18,399 INFO [M:0;303fe01965e8:45907 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T16:17:18,399 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T16:17:18,399 INFO [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:17:18,399 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:17:18,399 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T16:17:18,399 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:17:18,399 INFO [M:0;303fe01965e8:45907 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=796.54 KB heapSize=977.90 KB 2024-11-16T16:17:18,417 DEBUG [M:0;303fe01965e8:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/40f749d1825648c28a7766aa957fbb48 is 82, key is hbase:meta,,1/info:regioninfo/1731773657240/Put/seqid=0 2024-11-16T16:17:18,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742486_1662 (size=5672) 2024-11-16T16:17:18,822 INFO [M:0;303fe01965e8:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/40f749d1825648c28a7766aa957fbb48 2024-11-16T16:17:18,850 DEBUG [M:0;303fe01965e8:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7076dbad39654eb1ab332f34a4dbbfc3 is 2479, key is \x00\x00\x00\x00\x00\x00\x00`/proc:d/1731773745271/Put/seqid=0 2024-11-16T16:17:18,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742487_1663 (size=46039) 2024-11-16T16:17:19,254 INFO [M:0;303fe01965e8:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=795.99 KB at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7076dbad39654eb1ab332f34a4dbbfc3 2024-11-16T16:17:19,258 INFO [M:0;303fe01965e8:45907 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7076dbad39654eb1ab332f34a4dbbfc3 2024-11-16T16:17:19,276 DEBUG [M:0;303fe01965e8:45907 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf713a8f417b415a86f46c169814ac36 is 69, key is 303fe01965e8,37355,1731773652566/rs:state/1731773655940/Put/seqid=0 2024-11-16T16:17:19,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073742488_1664 (size=5156) 2024-11-16T16:17:19,679 INFO [M:0;303fe01965e8:45907 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2244 (bloomFilter=true), to=hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf713a8f417b415a86f46c169814ac36 2024-11-16T16:17:19,683 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/40f749d1825648c28a7766aa957fbb48 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/40f749d1825648c28a7766aa957fbb48 2024-11-16T16:17:19,685 INFO [M:0;303fe01965e8:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/40f749d1825648c28a7766aa957fbb48, entries=8, sequenceid=2244, filesize=5.5 K 2024-11-16T16:17:19,686 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7076dbad39654eb1ab332f34a4dbbfc3 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7076dbad39654eb1ab332f34a4dbbfc3 2024-11-16T16:17:19,688 INFO [M:0;303fe01965e8:45907 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7076dbad39654eb1ab332f34a4dbbfc3 2024-11-16T16:17:19,688 INFO [M:0;303fe01965e8:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7076dbad39654eb1ab332f34a4dbbfc3, entries=174, sequenceid=2244, filesize=45.0 K 2024-11-16T16:17:19,689 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/bf713a8f417b415a86f46c169814ac36 as hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf713a8f417b415a86f46c169814ac36 2024-11-16T16:17:19,692 INFO [M:0;303fe01965e8:45907 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46569/user/jenkins/test-data/0a15f166-8488-2122-3d51-af1338270050/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/bf713a8f417b415a86f46c169814ac36, entries=1, sequenceid=2244, filesize=5.0 K 2024-11-16T16:17:19,693 INFO [M:0;303fe01965e8:45907 {}] regionserver.HRegion(3140): Finished flush of dataSize ~796.54 KB/815659, heapSize ~977.60 KB/1001064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2244, compaction requested=false 2024-11-16T16:17:19,696 INFO [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T16:17:19,696 DEBUG [M:0;303fe01965e8:45907 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731773838399Disabling compacts and flushes for region at 1731773838399Disabling writes for close at 1731773838399Obtaining lock to block concurrent updates at 1731773838399Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731773838399Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=815659, getHeapSize=1001304, getOffHeapSize=0, getCellsCount=2443 at 1731773838399Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731773838400 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731773838400Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731773838417 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731773838417Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731773838831 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731773838850 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731773838850Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731773839258 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731773839275 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731773839275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@683b71bb: reopening flushed file at 1731773839682 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ac8634c: reopening flushed file at 1731773839685 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d082207: reopening flushed file at 1731773839689 (+4 ms)Finished flush of dataSize ~796.54 KB/815659, heapSize ~977.60 KB/1001064, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1293ms, sequenceid=2244, compaction requested=false at 1731773839693 (+4 ms)Writing region close event to WAL at 1731773839696 (+3 ms)Closed at 1731773839696 2024-11-16T16:17:19,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41113 is added to blk_1073741830_1006 (size=962734) 2024-11-16T16:17:19,700 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T16:17:19,700 INFO [M:0;303fe01965e8:45907 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T16:17:19,700 INFO [M:0;303fe01965e8:45907 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45907 2024-11-16T16:17:19,700 INFO [M:0;303fe01965e8:45907 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T16:17:19,814 INFO [M:0;303fe01965e8:45907 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T16:17:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T16:17:19,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45907-0x101446cd3870000, quorum=127.0.0.1:64212, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T16:17:19,818 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-16T16:17:19,820 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T16:17:19,820 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T16:17:19,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T16:17:19,820 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.log.dir/,STOPPED} 2024-11-16T16:17:19,822 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-16T16:17:19,822 WARN [BP-1064039519-172.17.0.2-1731773647624 heartbeating to localhost/127.0.0.1:46569 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-16T16:17:19,822 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-16T16:17:19,822 WARN [BP-1064039519-172.17.0.2-1731773647624 heartbeating to localhost/127.0.0.1:46569 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1064039519-172.17.0.2-1731773647624 (Datanode Uuid b63cc791-161c-4efb-b0a1-2b19fdb310d8) service to localhost/127.0.0.1:46569 2024-11-16T16:17:19,825 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data1/current/BP-1064039519-172.17.0.2-1731773647624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T16:17:19,825 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/cluster_e0b79bbc-f7f5-9a0a-45c6-e45b6b4cf0e7/data/data2/current/BP-1064039519-172.17.0.2-1731773647624 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-16T16:17:19,826 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-16T16:17:19,832 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-16T16:17:19,833 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-16T16:17:19,833 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-16T16:17:19,833 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-16T16:17:19,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/99bfaffb-5e5e-687f-88a9-446d0dbee883/hadoop.log.dir/,STOPPED} 2024-11-16T16:17:19,849 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T16:17:19,980 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down